Merge branch 'archweb-generic'parabolaweb-2015-04-17

25 files changed, 498 insertions, 453 deletions

diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index 8c17c78f..1f16a375 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -238,7 +238,7 @@ class MirrorCheckPool(object):
         for url in list(urls):
             self.tasks.put(url)
         self.threads = []
-        for i in range(num_threads):
+        for _ in range(num_threads):
             thread = Thread(target=mirror_url_worker,
                     args=(self.tasks, self.logs, location, timeout))
             thread.daemon = True
diff --git a/mirrors/urls.py b/mirrors/urls.py
index b1054380..fc510fbb 100644
--- a/mirrors/urls.py
+++ b/mirrors/urls.py
@@ -1,15 +1,18 @@
 from django.conf.urls import patterns
 
-urlpatterns = patterns('mirrors.views',
-    (r'^$',                     'mirrors', {}, 'mirror-list'),
-    (r'^status/$',              'status',  {}, 'mirror-status'),
-    (r'^status/json/$',         'status_json',  {}, 'mirror-status-json'),
-    (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'),
-    (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'),
-    (r'^locations/json/$',      'locations_json',  {}, 'mirror-locations-json'),
-    (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'),
-    (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'),
-    (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'),
+from .views import mirrors, status, mirror_details, url_details
+from .views.api import status_json, mirror_details_json, locations_json
+
+urlpatterns = patterns('',
+    (r'^$', mirrors, {}, 'mirror-list'),
+    (r'^status/$', status,  {}, 'mirror-status'),
+    (r'^status/json/$', status_json,  {}, 'mirror-status-json'),
+    (r'^status/tier/(?P<tier>\d+)/$', status, {}, 'mirror-status-tier'),
+    (r'^status/tier/(?P<tier>\d+)/json/$', status_json, {}, 'mirror-status-tier-json'),
+    (r'^locations/json/$', locations_json,  {}, 'mirror-locations-json'),
+    (r'^(?P<name>[\.\-\w]+)/$', mirror_details),
+    (r'^(?P<name>[\.\-\w]+)/json/$', mirror_details_json),
+    (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', url_details),
 )
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py
index bba54ec9..a64656a9 100644
--- a/mirrors/urls_mirrorlist.py
+++ b/mirrors/urls_mirrorlist.py
@@ -1,7 +1,7 @@
 from django.conf.urls import patterns
 
 
-urlpatterns = patterns('mirrors.views',
+urlpatterns = patterns('mirrors.views.mirrorlist',
     (r'^$',         'generate_mirrorlist', {}, 'mirrorlist'),
     (r'^all/$',     'find_mirrors', {'countries': ['all']}),
     (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple',
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 930adb8d..7c2f5d17 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -4,7 +4,6 @@ from django.db import connection
 from django.db.models import Count, Max, Min
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now
-from django_countries.fields import Country
 
 from main.utils import cache_function, database_vendor
 from .models import MirrorLog, MirrorUrl
@@ -159,9 +158,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
     cutoff_time = now() - cutoff
     errors = MirrorLog.objects.filter(
             is_success=False, check_time__gte=cutoff_time,
-            url__mirror__public=True).values(
-            'url__url', 'url__country', 'url__protocol__protocol',
-            'url__mirror__tier', 'error').annotate(
+            url__mirror__public=True).values('url__id', 'error').annotate(
             error_count=Count('error'), last_occurred=Max('check_time')
             ).order_by('-last_occurred', '-error_count')
 
@@ -172,8 +169,11 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
                 url__mirror__public=True)
 
     errors = list(errors)
+    to_fetch = [err['url__id'] for err in errors]
+    urls = MirrorUrl.objects.select_related(
+            'mirror', 'protocol').in_bulk(to_fetch)
     for err in errors:
-        err['country'] = Country(err['url__country'], flag_url='')
+        err['url'] = urls[err['url__id']]
     return errors
 
 
@@ -183,12 +183,12 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF):
     status data available, it is used to determine a good choice by looking at
     the last batch of status rows.'''
     cutoff_time = now() - cutoff
-    status_data = MirrorLog.objects.filter(
+    log_data = MirrorLog.objects.filter(
             check_time__gte=cutoff_time).aggregate(
             Max('check_time'), Max('last_sync'))
-    if status_data['check_time__max'] is not None:
-        min_check_time = status_data['check_time__max'] - timedelta(minutes=5)
-        min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20)
+    if log_data['check_time__max'] is not None:
+        min_check_time = log_data['check_time__max'] - timedelta(minutes=5)
+        min_sync_time = log_data['last_sync__max'] - timedelta(minutes=20)
         best_logs = MirrorLog.objects.select_related('url').filter(
                 is_success=True,
                 check_time__gte=min_check_time, last_sync__gte=min_sync_time,
diff --git a/mirrors/views.py b/mirrors/views.py
deleted file mode 100644
index 65fa0123..00000000
--- a/mirrors/views.py
+++ /dev/null
@@ -1,368 +0,0 @@
-from datetime import timedelta
-from itertools import groupby
-import json
-from operator import attrgetter, itemgetter
-
-from django import forms
-from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
-from django.core.serializers.json import DjangoJSONEncoder
-from django.db import connection
-from django.db.models import Q
-from django.http import Http404, HttpResponse
-from django.shortcuts import get_object_or_404, redirect, render
-from django.utils.timezone import now
-from django.views.decorators.cache import cache_page
-from django.views.decorators.csrf import csrf_exempt
-from django.views.decorators.http import condition
-from django_countries import countries
-from django_countries.fields import Country
-
-from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
-        CheckLocation)
-from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF
-
-
-class MirrorlistForm(forms.Form):
-    country = forms.MultipleChoiceField(required=False,
-            widget=SelectMultiple(attrs={'size': '12'}))
-    protocol = forms.MultipleChoiceField(required=False,
-            widget=CheckboxSelectMultiple)
-    ip_version = forms.MultipleChoiceField(required=False,
-            label="IP version", choices=(('4','IPv4'), ('6','IPv6')),
-            widget=CheckboxSelectMultiple)
-    use_mirror_status = forms.BooleanField(required=False)
-
-    def __init__(self, *args, **kwargs):
-        super(MirrorlistForm, self).__init__(*args, **kwargs)
-        fields = self.fields
-        fields['country'].choices = [('all','All')] + self.get_countries()
-        fields['country'].initial = ['all']
-        protos = [(p.protocol, p.protocol) for p in
-                MirrorProtocol.objects.filter(is_download=True)]
-        initial = MirrorProtocol.objects.filter(is_download=True, default=True)
-        fields['protocol'].choices = protos
-        fields['protocol'].initial = [p.protocol for p in initial]
-        fields['ip_version'].initial = ['4']
-
-    def get_countries(self):
-        country_codes = set()
-        country_codes.update(MirrorUrl.objects.filter(active=True,
-                mirror__active=True).exclude(country='').values_list(
-                'country', flat=True).order_by().distinct())
-        code_list = [(code, countries.name(code)) for code in country_codes]
-        return sorted(code_list, key=itemgetter(1))
-
-    def as_div(self):
-        "Returns this form rendered as HTML <divs>s."
-        return self._html_output(
-            normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
-            error_row = u'%s',
-            row_ender = '</div>',
-            help_text_html = u' <span class="helptext">%s</span>',
-            errors_on_separate_row = True)
-
-
-@csrf_exempt
-def generate_mirrorlist(request):
-    if request.method == 'POST' or len(request.GET) > 0:
-        form = MirrorlistForm(data=request.REQUEST)
-        if form.is_valid():
-            countries = form.cleaned_data['country']
-            protocols = form.cleaned_data['protocol']
-            use_status = form.cleaned_data['use_mirror_status']
-            ipv4 = '4' in form.cleaned_data['ip_version']
-            ipv6 = '6' in form.cleaned_data['ip_version']
-            return find_mirrors(request, countries, protocols,
-                    use_status, ipv4, ipv6)
-    else:
-        form = MirrorlistForm()
-
-    return render(request, 'mirrors/mirrorlist_generate.html',
-            {'mirrorlist_form': form})
-
-
-def status_filter(original_urls):
-    status_info = get_mirror_statuses()
-    scores = {u.id: u.score for u in status_info['urls']}
-    urls = []
-    for u in original_urls:
-        u.score = scores.get(u.id, None)
-        # also include mirrors that don't have an up to date score
-        # (as opposed to those that have been set with no score)
-        if (u.id not in scores) or (u.score and u.score < 100.0):
-            urls.append(u)
-    # if a url doesn't have a score, treat it as the highest possible
-    return sorted(urls, key=lambda x: x.score or 100.0)
-
-
-def find_mirrors(request, countries=None, protocols=None, use_status=False,
-        ipv4_supported=True, ipv6_supported=True):
-    if not protocols:
-        protocols = MirrorProtocol.objects.filter(is_download=True)
-    elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
-        # we already have a queryset, no need to query again
-        pass
-    else:
-        protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
-    qset = MirrorUrl.objects.select_related().filter(
-            protocol__in=protocols, active=True,
-            mirror__public=True, mirror__active=True)
-    if countries and 'all' not in countries:
-        qset = qset.filter(country__in=countries)
-
-    ip_version = Q()
-    if ipv4_supported:
-        ip_version |= Q(has_ipv4=True)
-    if ipv6_supported:
-        ip_version |= Q(has_ipv6=True)
-    qset = qset.filter(ip_version)
-
-    if not use_status:
-        sort_key = attrgetter('country.name', 'mirror.name', 'url')
-        urls = sorted(qset, key=sort_key)
-        template = 'mirrors/mirrorlist.txt'
-    else:
-        urls = status_filter(qset)
-        template = 'mirrors/mirrorlist_status.txt'
-
-    context = {
-        'mirror_urls': urls,
-    }
-    return render(request, template, context, content_type='text/plain')
-
-
-def find_mirrors_simple(request, protocol):
-    if protocol == 'smart':
-        return redirect('mirrorlist_simple', 'http', permanent=True)
-    proto = get_object_or_404(MirrorProtocol, protocol=protocol)
-    return find_mirrors(request, protocols=[proto])
-
-
-def mirrors(request):
-    mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
-    protos = MirrorUrl.objects.values_list(
-            'mirror_id', 'protocol__protocol').order_by(
-            'mirror_id', 'protocol__protocol').distinct()
-    countries = MirrorUrl.objects.values_list(
-            'mirror_id', 'country').order_by(
-            'mirror_id', 'country').distinct()
-
-    if not request.user.is_authenticated():
-        mirror_list = mirror_list.filter(public=True, active=True)
-        protos = protos.filter(
-                mirror__public=True, mirror__active=True, active=True)
-        countries = countries.filter(
-                mirror__public=True, mirror__active=True, active=True)
-
-    protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
-    countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))}
-
-    for mirror in mirror_list:
-        item_protos = protos.get(mirror.id, [])
-        mirror.protocols = [item[1] for item in item_protos]
-        mirror.country = None
-        item_countries = countries.get(mirror.id, [])
-        if len(item_countries) == 1:
-            mirror.country = Country(item_countries[0][1])
-
-    return render(request, 'mirrors/mirrors.html',
-            {'mirror_list': mirror_list})
-
-
-def mirror_details(request, name):
-    mirror = get_object_or_404(Mirror, name=name)
-    authorized = request.user.is_authenticated()
-    if not authorized and \
-            (not mirror.public or not mirror.active):
-        raise Http404
-    error_cutoff = timedelta(days=7)
-
-    status_info = get_mirror_statuses(mirror_id=mirror.id,
-            show_all=authorized)
-    checked_urls = {url for url in status_info['urls'] \
-            if url.mirror_id == mirror.id}
-    all_urls = mirror.urls.select_related('protocol')
-    if not authorized:
-        all_urls = all_urls.filter(active=True)
-    all_urls = set(all_urls)
-    # Add dummy data for URLs that we haven't checked recently
-    other_urls = all_urls.difference(checked_urls)
-    for url in other_urls:
-        for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
-                'duration_stddev', 'score'):
-            setattr(url, attr, None)
-    all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
-
-    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
-            show_all=True)
-
-    context = {
-        'mirror': mirror,
-        'urls': all_urls,
-        'cutoff': error_cutoff,
-        'error_logs': error_logs,
-    }
-    return render(request, 'mirrors/mirror_details.html', context)
-
-
-def mirror_details_json(request, name):
-    authorized = request.user.is_authenticated()
-    mirror = get_object_or_404(Mirror, name=name)
-    if not authorized and (not mirror.public or not mirror.active):
-        raise Http404
-    status_info = get_mirror_statuses(mirror_id=mirror.id,
-            show_all=authorized)
-    data = status_info.copy()
-    data['version'] = 3
-    to_json = json.dumps(data, ensure_ascii=False,
-            cls=ExtendedMirrorStatusJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-
-def url_details(request, name, url_id):
-    url = get_object_or_404(MirrorUrl.objects.select_related(),
-            id=url_id, mirror__name=name)
-    mirror = url.mirror
-    authorized = request.user.is_authenticated()
-    if not authorized and \
-            (not mirror.public or not mirror.active or not url.active):
-        raise Http404
-    error_cutoff = timedelta(days=7)
-    cutoff_time = now() - error_cutoff
-    logs = MirrorLog.objects.select_related('location').filter(
-            url=url, check_time__gte=cutoff_time).order_by('-check_time')
-
-    context = {
-        'url': url,
-        'logs': logs,
-    }
-    return render(request, 'mirrors/url_details.html', context)
-
-
-def status_last_modified(request, *args, **kwargs):
-    cursor = connection.cursor()
-    cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog")
-    return cursor.fetchone()[0]
-
-
-@condition(last_modified_func=status_last_modified)
-def status(request, tier=None):
-    if tier is not None:
-        tier = int(tier)
-        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
-            raise Http404
-    bad_timedelta = timedelta(days=3)
-    status_info = get_mirror_statuses()
-
-    urls = status_info['urls']
-    good_urls = []
-    bad_urls = []
-    for url in urls:
-        # screen by tier if we were asked to
-        if tier is not None and url.mirror.tier != tier:
-            continue
-        # split them into good and bad lists based on delay
-        if url.completion_pct is None:
-            # skip URLs that have never been checked
-            continue
-        elif not url.delay or url.delay > bad_timedelta:
-            bad_urls.append(url)
-        else:
-            good_urls.append(url)
-
-    error_logs = get_mirror_errors()
-    if tier is not None:
-        error_logs = [log for log in error_logs
-                if log['url__mirror__tier'] == tier]
-
-    context = status_info.copy()
-    context.update({
-        'good_urls': sorted(good_urls, key=attrgetter('score')),
-        'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
-        'error_logs': error_logs,
-        'tier': tier,
-    })
-    return render(request, 'mirrors/status.html', context)
-
-
-class MirrorStatusJSONEncoder(DjangoJSONEncoder):
-    '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
-    serialization. The base class takes care of datetime.datetime types.'''
-    url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
-            'delay', 'duration_avg', 'duration_stddev', 'score')
-
-    def default(self, obj):
-        if isinstance(obj, timedelta):
-            # always returned as integer seconds
-            return obj.days * 24 * 3600 + obj.seconds
-        if isinstance(obj, MirrorUrl):
-            data = {attr: getattr(obj, attr) for attr in self.url_attributes}
-            country = obj.country
-            data['country'] = unicode(country.name)
-            data['country_code'] = country.code
-            return data
-        if isinstance(obj, MirrorProtocol):
-            return unicode(obj)
-        return super(MirrorStatusJSONEncoder, self).default(obj)
-
-
-class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
-    '''Adds URL check history information.'''
-    log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
-            'location_id')
-
-    def default(self, obj):
-        if isinstance(obj, MirrorUrl):
-            data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
-            cutoff = now() - DEFAULT_CUTOFF
-            data['logs'] = list(obj.logs.filter(
-                    check_time__gte=cutoff).order_by('check_time'))
-            return data
-        if isinstance(obj, MirrorLog):
-            return {attr: getattr(obj, attr) for attr in self.log_attributes}
-        return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
-
-
-@cache_page(67)
-@condition(last_modified_func=status_last_modified)
-def status_json(request, tier=None):
-    if tier is not None:
-        tier = int(tier)
-        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
-            raise Http404
-    status_info = get_mirror_statuses()
-    data = status_info.copy()
-    if tier is not None:
-        data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
-    data['version'] = 3
-    to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-
-class LocationJSONEncoder(DjangoJSONEncoder):
-    '''Base JSONEncoder extended to handle CheckLocation objects.'''
-
-    def default(self, obj):
-        if isinstance(obj, CheckLocation):
-            return {
-                'id': obj.pk,
-                'hostname': obj.hostname,
-                'source_ip': obj.source_ip,
-                'country': unicode(obj.country.name),
-                'country_code': obj.country.code,
-                'ip_version': obj.ip_version,
-            }
-        return super(LocationJSONEncoder, self).default(obj)
-
-
-def locations_json(request):
-    data = {}
-    data['version'] = 1
-    data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
-    to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py
new file mode 100644
index 00000000..01e8519d
--- /dev/null
+++ b/mirrors/views/__init__.py
@@ -0,0 +1,149 @@
+from datetime import timedelta
+from itertools import groupby
+from operator import attrgetter, itemgetter
+
+from django.db import connection
+from django.http import Http404
+from django.shortcuts import get_object_or_404, render
+from django.utils.timezone import now
+from django.views.decorators.http import condition
+from django_countries.fields import Country
+
+from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+        CheckLocation)
+from ..utils import get_mirror_statuses, get_mirror_errors
+
+
+def mirrors(request):
+    mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
+    protos = MirrorUrl.objects.values_list(
+            'mirror_id', 'protocol__protocol').order_by(
+            'mirror_id', 'protocol__protocol').distinct()
+    countries = MirrorUrl.objects.values_list(
+            'mirror_id', 'country').order_by(
+            'mirror_id', 'country').distinct()
+
+    if not request.user.is_authenticated():
+        mirror_list = mirror_list.filter(public=True, active=True)
+        protos = protos.filter(
+                mirror__public=True, mirror__active=True, active=True)
+        countries = countries.filter(
+                mirror__public=True, mirror__active=True, active=True)
+
+    protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
+    countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))}
+
+    for mirror in mirror_list:
+        item_protos = protos.get(mirror.id, [])
+        mirror.protocols = [item[1] for item in item_protos]
+        mirror.country = None
+        item_countries = countries.get(mirror.id, [])
+        if len(item_countries) == 1:
+            mirror.country = Country(item_countries[0][1])
+
+    return render(request, 'mirrors/mirrors.html',
+            {'mirror_list': mirror_list})
+
+
+def mirror_details(request, name):
+    mirror = get_object_or_404(Mirror, name=name)
+    authorized = request.user.is_authenticated()
+    if not authorized and \
+            (not mirror.public or not mirror.active):
+        raise Http404
+    error_cutoff = timedelta(days=7)
+
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
+    checked_urls = {url for url in status_info['urls'] \
+            if url.mirror_id == mirror.id}
+    all_urls = mirror.urls.select_related('protocol')
+    if not authorized:
+        all_urls = all_urls.filter(active=True)
+    all_urls = set(all_urls)
+    # Add dummy data for URLs that we haven't checked recently
+    other_urls = all_urls.difference(checked_urls)
+    for url in other_urls:
+        for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
+                'duration_stddev', 'score'):
+            setattr(url, attr, None)
+    all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
+
+    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
+            show_all=True)
+
+    context = {
+        'mirror': mirror,
+        'urls': all_urls,
+        'cutoff': error_cutoff,
+        'error_logs': error_logs,
+    }
+    return render(request, 'mirrors/mirror_details.html', context)
+
+
+def url_details(request, name, url_id):
+    url = get_object_or_404(MirrorUrl.objects.select_related(),
+            id=url_id, mirror__name=name)
+    mirror = url.mirror
+    authorized = request.user.is_authenticated()
+    if not authorized and \
+            (not mirror.public or not mirror.active or not url.active):
+        raise Http404
+    error_cutoff = timedelta(days=7)
+    cutoff_time = now() - error_cutoff
+    logs = MirrorLog.objects.select_related('location').filter(
+            url=url, check_time__gte=cutoff_time).order_by('-check_time')
+
+    context = {
+        'url': url,
+        'logs': logs,
+    }
+    return render(request, 'mirrors/url_details.html', context)
+
+
+def status_last_modified(request, *args, **kwargs):
+    cursor = connection.cursor()
+    cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog")
+    return cursor.fetchone()[0]
+
+
+@condition(last_modified_func=status_last_modified)
+def status(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
+    bad_timedelta = timedelta(days=3)
+    status_info = get_mirror_statuses()
+
+    urls = status_info['urls']
+    good_urls = []
+    bad_urls = []
+    for url in urls:
+        # screen by tier if we were asked to
+        if tier is not None and url.mirror.tier != tier:
+            continue
+        # split them into good and bad lists based on delay
+        if url.completion_pct is None:
+            # skip URLs that have never been checked
+            continue
+        elif not url.delay or url.delay > bad_timedelta:
+            bad_urls.append(url)
+        else:
+            good_urls.append(url)
+
+    error_logs = get_mirror_errors()
+    if tier is not None:
+        error_logs = [log for log in error_logs
+                if log['url'].mirror.tier == tier]
+
+    context = status_info.copy()
+    context.update({
+        'good_urls': sorted(good_urls, key=attrgetter('score')),
+        'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
+        'error_logs': error_logs,
+        'tier': tier,
+    })
+    return render(request, 'mirrors/status.html', context)
+
+# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/api.py b/mirrors/views/api.py
new file mode 100644
index 00000000..b72585e6
--- /dev/null
+++ b/mirrors/views/api.py
@@ -0,0 +1,108 @@
+from datetime import timedelta
+import json
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.http import Http404, HttpResponse
+from django.shortcuts import get_object_or_404
+from django.utils.timezone import now
+
+from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+        CheckLocation)
+from ..utils import get_mirror_statuses, DEFAULT_CUTOFF
+
+
+class MirrorStatusJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
+    serialization. The base class takes care of datetime.datetime types.'''
+    url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
+            'delay', 'duration_avg', 'duration_stddev', 'score')
+
+    def default(self, obj):
+        if isinstance(obj, timedelta):
+            # always returned as integer seconds
+            return obj.days * 24 * 3600 + obj.seconds
+        if isinstance(obj, MirrorUrl):
+            data = {attr: getattr(obj, attr) for attr in self.url_attributes}
+            country = obj.country
+            data['country'] = unicode(country.name)
+            data['country_code'] = country.code
+            return data
+        if isinstance(obj, MirrorProtocol):
+            return unicode(obj)
+        return super(MirrorStatusJSONEncoder, self).default(obj)
+
+
+class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
+    '''Adds URL check history information.'''
+    log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
+            'location_id')
+
+    def default(self, obj):
+        if isinstance(obj, MirrorUrl):
+            data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+            cutoff = now() - DEFAULT_CUTOFF
+            data['logs'] = list(obj.logs.filter(
+                    check_time__gte=cutoff).order_by('check_time'))
+            return data
+        if isinstance(obj, MirrorLog):
+            data = {attr: getattr(obj, attr) for attr in self.log_attributes}
+            data['error'] = obj.error or None
+            return data
+        return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+
+
+class LocationJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle CheckLocation objects.'''
+
+    def default(self, obj):
+        if isinstance(obj, CheckLocation):
+            return {
+                'id': obj.pk,
+                'hostname': obj.hostname,
+                'source_ip': obj.source_ip,
+                'country': unicode(obj.country.name),
+                'country_code': obj.country.code,
+                'ip_version': obj.ip_version,
+            }
+        return super(LocationJSONEncoder, self).default(obj)
+
+
+def status_json(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
+    status_info = get_mirror_statuses()
+    data = status_info.copy()
+    if tier is not None:
+        data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
+    data['version'] = 3
+    to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+def mirror_details_json(request, name):
+    authorized = request.user.is_authenticated()
+    mirror = get_object_or_404(Mirror, name=name)
+    if not authorized and (not mirror.public or not mirror.active):
+        raise Http404
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
+    data = status_info.copy()
+    data['version'] = 3
+    to_json = json.dumps(data, ensure_ascii=False,
+            cls=ExtendedMirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+def locations_json(request):
+    data = {}
+    data['version'] = 1
+    data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
+    to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py
new file mode 100644
index 00000000..3c68d036
--- /dev/null
+++ b/mirrors/views/mirrorlist.py
@@ -0,0 +1,129 @@
+from operator import attrgetter, itemgetter
+
+from django import forms
+from django.db.models import Q
+from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
+from django.shortcuts import get_object_or_404, redirect, render
+from django.views.decorators.csrf import csrf_exempt
+from django_countries import countries
+
+from ..models import MirrorUrl, MirrorProtocol
+from ..utils import get_mirror_statuses
+
+
+class MirrorlistForm(forms.Form):
+    country = forms.MultipleChoiceField(required=False,
+            widget=SelectMultiple(attrs={'size': '12'}))
+    protocol = forms.MultipleChoiceField(required=False,
+            widget=CheckboxSelectMultiple)
+    ip_version = forms.MultipleChoiceField(required=False,
+            label="IP version", choices=(('4','IPv4'), ('6','IPv6')),
+            widget=CheckboxSelectMultiple)
+    use_mirror_status = forms.BooleanField(required=False)
+
+    def __init__(self, *args, **kwargs):
+        super(MirrorlistForm, self).__init__(*args, **kwargs)
+        fields = self.fields
+        fields['country'].choices = [('all','All')] + self.get_countries()
+        fields['country'].initial = ['all']
+        protos = [(p.protocol, p.protocol) for p in
+                MirrorProtocol.objects.filter(is_download=True)]
+        initial = MirrorProtocol.objects.filter(is_download=True, default=True)
+        fields['protocol'].choices = protos
+        fields['protocol'].initial = [p.protocol for p in initial]
+        fields['ip_version'].initial = ['4']
+
+    def get_countries(self):
+        country_codes = set()
+        country_codes.update(MirrorUrl.objects.filter(active=True,
+                mirror__active=True).exclude(country='').values_list(
+                'country', flat=True).order_by().distinct())
+        code_list = [(code, countries.name(code)) for code in country_codes]
+        return sorted(code_list, key=itemgetter(1))
+
+    def as_div(self):
+        "Returns this form rendered as HTML <divs>s."
+        return self._html_output(
+            normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+            error_row = u'%s',
+            row_ender = '</div>',
+            help_text_html = u' <span class="helptext">%s</span>',
+            errors_on_separate_row = True)
+
+
+@csrf_exempt
+def generate_mirrorlist(request):
+    if request.method == 'POST' or len(request.GET) > 0:
+        form = MirrorlistForm(data=request.REQUEST)
+        if form.is_valid():
+            countries = form.cleaned_data['country']
+            protocols = form.cleaned_data['protocol']
+            use_status = form.cleaned_data['use_mirror_status']
+            ipv4 = '4' in form.cleaned_data['ip_version']
+            ipv6 = '6' in form.cleaned_data['ip_version']
+            return find_mirrors(request, countries, protocols,
+                    use_status, ipv4, ipv6)
+    else:
+        form = MirrorlistForm()
+
+    return render(request, 'mirrors/mirrorlist_generate.html',
+            {'mirrorlist_form': form})
+
+
+def status_filter(original_urls):
+    status_info = get_mirror_statuses()
+    scores = {u.id: u.score for u in status_info['urls']}
+    urls = []
+    for u in original_urls:
+        u.score = scores.get(u.id, None)
+        # also include mirrors that don't have an up to date score
+        # (as opposed to those that have been set with no score)
+        if (u.id not in scores) or (u.score and u.score < 100.0):
+            urls.append(u)
+    # if a url doesn't have a score, treat it as the highest possible
+    return sorted(urls, key=lambda x: x.score or 100.0)
+
+
+def find_mirrors(request, countries=None, protocols=None, use_status=False,
+        ipv4_supported=True, ipv6_supported=True):
+    if not protocols:
+        protocols = MirrorProtocol.objects.filter(is_download=True)
+    elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
+        # we already have a queryset, no need to query again
+        pass
+    else:
+        protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
+    qset = MirrorUrl.objects.select_related().filter(
+            protocol__in=protocols, active=True,
+            mirror__public=True, mirror__active=True)
+    if countries and 'all' not in countries:
+        qset = qset.filter(country__in=countries)
+
+    ip_version = Q()
+    if ipv4_supported:
+        ip_version |= Q(has_ipv4=True)
+    if ipv6_supported:
+        ip_version |= Q(has_ipv6=True)
+    qset = qset.filter(ip_version)
+
+    if not use_status:
+        sort_key = attrgetter('country.name', 'mirror.name', 'url')
+        urls = sorted(qset, key=sort_key)
+        template = 'mirrors/mirrorlist.txt'
+    else:
+        urls = status_filter(qset)
+        template = 'mirrors/mirrorlist_status.txt'
+
+    context = {
+        'mirror_urls': urls,
+    }
+    return render(request, template, context, content_type='text/plain')
+
+
+def find_mirrors_simple(request, protocol):
+    if protocol == 'smart':
+        return redirect('mirrorlist_simple', 'http', permanent=True)
+    proto = get_object_or_404(MirrorProtocol, protocol=protocol)
+    return find_mirrors(request, protocols=[proto])
+
+# vim: set ts=4 sw=4 et:
diff --git a/newrelic.ini b/newrelic.ini
index 98cca5b6..72158dc4 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -54,7 +54,7 @@ monitor_mode = true
 # write out a log file, it is also possible to say "stderr" and
 # output to standard error output. This would normally result in
 # output appearing in your web server log.
-log_file = /tmp/newrelic-python-agent.log
+#log_file = /tmp/newrelic-python-agent.log
 
 # Sets the level of detail of messages sent to the log file, if
 # a log file location has been provided. Possible values, in
@@ -68,14 +68,27 @@ log_file = /tmp/newrelic-python-agent.log
 log_level = info
 
 # The Python Agent communicates with the New Relic service using
-# HTTP by default. If you want to communicate via HTTPS to
-# increase security, then turn on SSL by setting this value to
-# true. Note, this will result in increased CPU overhead to
-# perform the encryption involved in SSL communication, but this
-# work is done asynchronously to the threads that process your
-# application code, so it should not impact response times.
+# SSL by default. Note that this does result in an increase in
+# CPU overhead, over and above what would occur for a non SSL
+# connection, to perform the encryption involved in the SSL
+# communication. This work is though done in a distinct thread
+# to those handling your web requests, so it should not impact
+# response times. You can if you wish revert to using a non SSL
+# connection, but this will result in information being sent
+# over a plain socket connection and will not be as secure.
 ssl = true
 
+# High Security Mode enforces certain security settings, and
+# prevents them from being overridden, so that no sensitive data
+# is sent to New Relic. Enabling High Security Mode means that
+# SSL is turned on, request parameters are not collected, and SQL
+# can not be sent to New Relic in its raw form. To activate High
+# Security Mode, it must be set to 'true' in this local .ini
+# configuration file AND be set to 'true' in the server-side
+# configuration in the New Relic user interface. For details, see
+# https://docs.newrelic.com/docs/subscriptions/high-security
+high_security = false
+
 # The Python Agent will attempt to connect directly to the New
 # Relic service. If there is an intermediate firewall between
 # your host and the New Relic service that requires you to use a
@@ -83,7 +96,12 @@ ssl = true
 # "proxy_port" settings to the required values for the HTTP
 # proxy. The "proxy_user" and "proxy_pass" settings should
 # additionally be set if proxy authentication is implemented by
-# the HTTP proxy.
+# the HTTP proxy. The "proxy_scheme" setting dictates what
+# protocol scheme is used in talking to the HTTP proxy. This
+# would normally always be set as "http" which will result in the
+# agent then using a SSL tunnel through the HTTP proxy for end to
+# end encryption.
+# proxy_scheme = http
 # proxy_host = hostname
 # proxy_port = 8080
 # proxy_user =
@@ -153,13 +171,13 @@ error_collector.enabled = true
 # To stop specific errors from reporting to the UI, set this to
 # a space separated list of the Python exception type names to
 # ignore. The exception name should be of the form 'module:class'.
-error_collector.ignore_errors = django.http.response:Http404
+error_collector.ignore_errors =
 
 # Browser monitoring is the Real User Monitoring feature of the UI.
 # For those Python web frameworks that are supported, this
 # setting enables the auto-insertion of the browser monitoring
 # JavaScript fragments.
-browser_monitoring.auto_instrument = true
+browser_monitoring.auto_instrument = false
 
 # A thread profiling session can be scheduled via the UI when
 # this option is enabled. The thread profiler will periodically
diff --git a/packages/views/search.py b/packages/views/search.py
index e4cd0423..6e892251 100644
--- a/packages/views/search.py
+++ b/packages/views/search.py
@@ -42,11 +42,13 @@ class PackageSearchForm(forms.Form):
         people = User.objects.filter(
                 is_active=True, userprofile__id__in=profile_ids).order_by(
                 'first_name', 'last_name')
-        people = [('', 'All'), ('orphan', 'Orphan')] + \
+        maintainers = [('', 'All'), ('orphan', 'Orphan')] + \
+                 [(p.username, p.get_full_name()) for p in people]
+        packagers = [('', 'All'), ('unknown', 'Unknown')] + \
                  [(p.username, p.get_full_name()) for p in people]
 
-        self.fields['maintainer'].choices = people
-        self.fields['packager'].choices = people
+        self.fields['maintainer'].choices = maintainers
+        self.fields['packager'].choices = packagers
 
     def exact_matches(self):
         # only do exact match search if 'q' is sole parameter
diff --git a/requirements.txt b/requirements.txt
index 54cf90f0..7b8c4822 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,12 @@
 -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
-Django==1.7.3
+Django==1.7.7
 IPy==0.81
 Jinja2==2.7.3
-Markdown==2.5.2
+Markdown==2.6.1
 MarkupSafe==0.23
 bencode==1.0
-django-jinja==1.0.5
-django_countries==3.0.2
-jsmin==2.1.0
+django-countries==3.3
+django-jinja==1.3.2
+jsmin==2.1.1
 pgpdump==1.5
-pytz>=2014.10
+pytz>=2015.2
diff --git a/requirements_prod.txt b/requirements_prod.txt
index ac528ef9..3396ebdb 100644
--- a/requirements_prod.txt
+++ b/requirements_prod.txt
@@ -1,15 +1,15 @@
 -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
-Django==1.7.3
+Django==1.7.7
 IPy==0.81
 Jinja2==2.7.3
-Markdown==2.5.2
+Markdown==2.6.1
 MarkupSafe==0.23
 bencode==1.0
-django-jinja==1.0.5
-django_countries==3.0.2
-jsmin==2.1.0
+django-countries==3.3
+django-jinja==1.3.2
+jsmin==2.1.1
 pgpdump==1.5
-psycopg2==2.5.4
+psycopg2==2.6
 pyinotify==0.9.5
-python-memcached==1.53
-pytz>=2014.10
+python-memcached==1.54
+pytz>=2015.2
diff --git a/settings.py b/settings.py
index fa27b2f2..70da44b8 100644
--- a/settings.py
+++ b/settings.py
@@ -152,11 +152,10 @@ LOGGING = {
             'handlers': ['mail_admins'],
             'level': 'ERROR',
             'propagate': True,
-        },
+        }
     },
 }
 
-
 ## Server used for linking to PGP keysearch results
 PGP_SERVER = 'pgp.mit.edu:11371'
 
@@ -199,9 +198,10 @@ BRANDING_WIKINAME = 'ParabolaWiki'
 BRANDING_EMAIL = 'Parabola Website Notification <nobody@parabola.nu>'
 BRANDING_OSEARCH_TAGS = 'gnu linuxlibre parabola package software'
 
-# Country name overrides for display purposes
+# Shorten some names just a bit
 COUNTRIES_OVERRIDE = {
-    'MK': 'Macedonia',
+    'GB': 'United Kingdom',
+    'US': 'United States',
 }
 
 ## Import local settings
diff --git a/sitemaps.py b/sitemaps.py
index 03ad9254..7746ecab 100644
--- a/sitemaps.py
+++ b/sitemaps.py
@@ -37,12 +37,17 @@ class PackagesSitemap(Sitemap):
 class PackageFilesSitemap(PackagesSitemap):
     changefreq = "weekly"
     priority = "0.1"
+    # we fixed a bug on the package files page on this day, force modification
+    lastmod_min = datetime(2015, 4, 12).replace(tzinfo=utc)
 
     def location(self, obj):
         return PackagesSitemap.location(self, obj) + 'files/'
 
     def lastmod(self, obj):
-        return obj.files_last_update
+        update = obj.files_last_update
+        if update is None:
+            return None
+        return max(update, self.lastmod_min)
 
 
 class PackageGroupsSitemap(Sitemap):
diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css
index db053a52..22f23b3a 100644
--- a/sitestatic/archweb.css
+++ b/sitestatic/archweb.css
@@ -377,6 +377,10 @@ ul.errorlist {
     color: red;
 }
 
+form ul.errorlist {
+    margin: 0.5em 0;
+}
+
 /* JS sorting via tablesorter */
 table th.tablesorter-header {
     padding-right: 20px;
diff --git a/templates/mirrors/error_table.html.jinja b/templates/mirrors/error_table.html.jinja
index 52f68135..132aae63 100644
--- a/templates/mirrors/error_table.html.jinja
+++ b/templates/mirrors/error_table.html.jinja
@@ -7,16 +7,18 @@
             <th>Error Message</th>
             <th>Last Occurred</th>
             <th>Occurrences (last {{ cutoff|hours }})</th>
+            <th></th>
         </tr>
     </thead>
     <tbody>
         {% for log in error_logs %}<tr class="{{ loop.cycle('odd', 'even') }}">
-            <td>{{ log.url__url }}</td>
-            <td>{{ log.url__protocol__protocol }}</td>
-            <td class="country">{{ country_flag(log.country) }}{{ log.country.name }}</td>
+            <td>{{ log.url.url }}</td>
+            <td>{{ log.url.protocol.protocol }}</td>
+            <td class="country">{{ country_flag(log.url.country) }}{{ log.url.country.name }}</td>
             <td class="wrap">{{ log.error|linebreaksbr }}</td>
             <td>{{ log.last_occurred|date('Y-m-d H:i') }}</td>
             <td>{{ log.error_count }}</td>
+            <td><a href="{{ log.url.get_absolute_url() }}">details</a></td>
         </tr>{% endfor %}
     </tbody>
 </table>
diff --git a/templates/mirrors/url_details_logs.html.jinja b/templates/mirrors/url_details_logs.html.jinja
index 58f179d8..51f54931 100644
--- a/templates/mirrors/url_details_logs.html.jinja
+++ b/templates/mirrors/url_details_logs.html.jinja
@@ -14,8 +14,8 @@
     <tbody>
         {% for log in logs %}<tr class="{{ loop.cycle('odd', 'even') }}">
             <td>{{ log.check_time|date('Y-m-d H:i') }}</td>
-            <td class="country">{{ country_flag(log.location.country) }}{{ log.location.country.name }}</td>
-            <td>{{ log.location.source_ip }}</td>
+            <td class="country">{% if log.location %}{{ country_flag(log.location.country) }}{{ log.location.country.name }}{% else %}Unknown{% endif %}</td>
+            <td>{% if log.location %}{{ log.location.source_ip }}{% else %}Unknown{% endif %}</td>
             <td>{{ log.last_sync|date('Y-m-d H:i') }}</td>
             <td>{{ log.delay|duration }}</td>
             <td>{{ log.duration|floatvalue }}</td>
diff --git a/templates/packages/details.html.jinja b/templates/packages/details.html.jinja
index 22c778cb..047bd8f6 100644
--- a/templates/packages/details.html.jinja
+++ b/templates/packages/details.html.jinja
@@ -108,7 +108,7 @@
         {% endif %}
         <tr>
             <th>Description:</th>
-            <td class="wrap" itemprop="description">{{ pkg.pkgdesc|default("") }}</td>
+            <td class="wrap" itemprop="description">{{ pkg.pkgdesc|default("", true) }}</td>
         </tr><tr>
             <th>Upstream URL:</th>
             <td>{% if pkg.url %}<a itemprop="url" href="{{ pkg.url }}"
@@ -191,7 +191,7 @@
         {% if user.is_authenticated() %}{% with flag_request = pkg.flag_request() %}{% if flag_request %}<tr>
             <th>Last Flag Request:</th>
             <td class="wrap">From {{ flag_request.who() }} on {{ flag_request.created|date }}:<br/>
-                <div class="userdata">{{ flag_request.message|linebreaksbr|default("{no message}") }}</div></td>
+                <div class="userdata">{{ flag_request.message|linebreaksbr|default("{no message}", true) }}</div></td>
         </tr>{% endif %}{% endwith %}{% endif %}
     </table>
     </div>
diff --git a/templates/packages/details_depend.html.jinja b/templates/packages/details_depend.html.jinja
index 404793b6..a2d3a010 100644
--- a/templates/packages/details_depend.html.jinja
+++ b/templates/packages/details_depend.html.jinja
@@ -1,8 +1,8 @@
 {% import 'packages/details_link.html.jinja' as details %}<li>{% if depend.pkg == None %}
-{% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }} <span class="virtual-dep">({% for pkg in depend.providers %}{{ details.details_link(pkg) }}{% if not loop.last %}, {% endif %}{% endfor %})</span>
-{% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }} <span class="virtual-dep">(virtual)</span>
+{% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }} <span class="virtual-dep">({% for pkg in depend.providers %}{{ details.details_link(pkg) }}{% if not loop.last %}, {% endif %}{% endfor %})</span>
+{% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }} <span class="virtual-dep">(virtual)</span>
 {% endif %}{% else %}
-{{ details.details_link(depend.pkg) }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }}
+{{ details.details_link(depend.pkg) }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }}
 {% if depend.pkg.repo.testing %} <span class="testing-dep"> (testing)</span>
 {% endif %}{% if depend.pkg.repo.staging %} <span class="staging-dep"> (staging)</span>
 {% endif %}{% endif %}
diff --git a/templates/packages/details_relatedto.html.jinja b/templates/packages/details_relatedto.html.jinja
index 955fdd37..818224de 100644
--- a/templates/packages/details_relatedto.html.jinja
+++ b/templates/packages/details_relatedto.html.jinja
@@ -1,3 +1,3 @@
 {% import 'packages/details_link.html.jinja' as details %}{% for related in all_related %}{% with best_satisfier = related.get_best_satisfier() %}
-<span class="related">{% if best_satisfier == None %}{{ related.name }}{% else %}{{ details.details_link(best_satisfier) }}{% endif %}{{ related.comparison|default("") }}{{ related.version|default("") }}{% if not loop.last %}, {% endif %}</span>
+<span class="related">{% if best_satisfier == None %}{{ related.name }}{% else %}{{ details.details_link(best_satisfier) }}{% endif %}{{ related.comparison|default('', true) }}{{ related.version|default('', true) }}{% if not loop.last %}, {% endif %}</span>
 {% endwith %}{% endfor %}
diff --git a/templates/packages/files_list.html.jinja b/templates/packages/files_list.html.jinja
index c8fc3b1a..ab3e1210 100644
--- a/templates/packages/files_list.html.jinja
+++ b/templates/packages/files_list.html.jinja
@@ -6,7 +6,7 @@ of the package; it may be out of date.</p>
 {% if files|length %}
 <ul>
 {% for file in files %}
-<li class="{% if file.is_directory %}d{% else %}f{% endif %}">{{ file.directory }}{{ file.filename|default('') }}</li>{% endfor %}
+<li class="{% if file.is_directory %}d{% else %}f{% endif %}">{{ file.directory }}{{ file.filename|default('', true) }}</li>{% endfor %}
 </ul>
 {% else %}
 <p class="message">Package has no files.</p>
diff --git a/templates/registration/login.html b/templates/registration/login.html
index c722527d..ff360de3 100644
--- a/templates/registration/login.html
+++ b/templates/registration/login.html
@@ -5,21 +5,15 @@
 
 {% block content %}
 <div id="dev-login" class="box">
-
     <h2>Developer Login</h2>
 
-    {% if form.has_errors %}
-    <p class="login-error">Your username and password didn't match. Please try again.</p>
-    {% endif %}
-
     <form id="dev-login-form" method="post">{% csrf_token %}
         <fieldset>
-            <legend>Enter login credentials</legend>
+            <legend>Please enter your credentials to login.</legend>
             {{ form.as_p }}
-            <p><label></label> <input type="submit" value="Login" /></p>
+            <p><label></label><input type="submit" value="Login"/></p>
         </fieldset>
     </form>
-
 </div>
 {% endblock %}
 
diff --git a/templates/registration/logout.html b/templates/registration/logout.html
index 71daa2e8..5c296c5d 100644
--- a/templates/registration/logout.html
+++ b/templates/registration/logout.html
@@ -1,11 +1,12 @@
 {% extends "base.html" %}
+
 {% block title %}{{ BRANDING_DISTRONAME }} - Logout successful{% endblock %}
 
 {% block content %}
 <div id="dev-logout" class="box">
     <h2>Developer Logout</h2>
 
-    <p>Logout was successful.<p>
+    <p>Logout was successful.
+    <a href='{% url 'login' %}'>Click here to login again.</a></p>
 </div>
 {% endblock %}
-
diff --git a/todolists/views.py b/todolists/views.py
index db6f20f0..a0b56e25 100644
--- a/todolists/views.py
+++ b/todolists/views.py
@@ -1,4 +1,5 @@
 import json
+from operator import attrgetter
 
 from django import forms
 from django.http import HttpResponse
@@ -224,8 +225,9 @@ def send_todolist_emails(todo_list, new_packages):
                 maint_packages.setdefault(maint, []).append(todo_package)
 
     for maint, packages in maint_packages.iteritems():
+        packages = sorted(packages, key=attrgetter('pkgname', 'arch'))
         ctx = Context({
-            'todo_packages': sorted(packages),
+            'todo_packages': packages,
             'todolist': todo_list,
         })
         template = loader.get_template('todolists/email_notification.txt')
diff --git a/urls.py b/urls.py
index 81c80ec6..c60a3dbe 100644
--- a/urls.py
+++ b/urls.py
@@ -89,12 +89,10 @@ urlpatterns += patterns('',
         'news-sitemap'),
 )
 
-# Authentication / Admin
+# Authentication
 urlpatterns += patterns('django.contrib.auth.views',
-    (r'^login/$',           'login',  {
-        'template_name': 'registration/login.html'}),
-    (r'^logout/$',          'logout', {
-        'template_name': 'registration/logout.html'}),
+    (r'^login/$', 'login', {'template_name': 'registration/login.html'}, 'login'),
+    (r'^logout/$', 'logout', {'template_name': 'registration/logout.html'}, 'logout'),
 )
 
 # Redirects for older known pages we see in the logs
@@ -116,10 +114,8 @@ legacy_urls = (
 
     ('^docs/en/guide/install/arch-install-guide.html',
         'https://wiki.parabola.nu/Installation_Guide'),
-    ('^docs/en/',
-        'https://wiki.parabola.nu/'),
-    ('^docs/',
-        'https://wiki.parabola.nu/'),
+    ('^docs/en/', 'https://wiki.parabola.nu/'),
+    ('^docs/', 'https://wiki.parabola.nu/'),
 
     ('^developers/$',   '/hackers/'),
     ('^trustedusers/$', '/hackers/'),