diff options
64 files changed, 507 insertions, 205 deletions
diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py index 10e6cfcb..b1f29d77 100644 --- a/devel/management/commands/pgp_import.py +++ b/devel/management/commands/pgp_import.py @@ -95,6 +95,7 @@ def parse_keydata(data): # parse all of the output from our successful GPG command logger.info("parsing command output") + node = None for line in data.split('\n'): parts = line.split(':') if parts[0] == 'pub': diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 1e456c8c..3e835f7c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -118,13 +118,9 @@ class RepoPackage(object): builddate = datetime.utcfromtimestamp(int(v[0])) self.builddate = builddate.replace(tzinfo=utc) except ValueError: - try: - self.builddate = datetime.strptime(v[0], - '%a %b %d %H:%M:%S %Y') - except ValueError: - logger.warning( - 'Package %s had unparsable build date %s', - self.name, v[0]) + logger.warning( + 'Package %s had unparsable build date %s', + self.name, v[0]) elif k == 'files': self.files = tuple(v) self.has_files = True diff --git a/devel/models.py b/devel/models.py index 67de40a6..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,6 @@ import pytz from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField diff --git a/devel/utils.py b/devel/utils.py index e8e3a6c4..340841f5 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -131,7 +131,7 @@ class UserFinder(object): self.username_email, self.user_name) for matcher in find_methods: user = matcher(name, email) - if user != None: + if user is not None: break self.cache[userstring] = user diff --git a/devel/views.py b/devel/views.py index 61c1e568..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -34,7 +34,7 @@ from .utils import get_annotated_maintainers @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() diff --git a/main/log.py b/main/log.py index 63634874..5c745cc8 100644 --- a/main/log.py +++ b/main/log.py @@ -46,7 +46,6 @@ class RateLimitFilter(object): trace = '\n'.join(traceback.format_exception(*record.exc_info)) key = md5(trace).hexdigest() - duplicate = False cache = self.cache_module.cache # Test if the cache works diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py index 0887b28c..7da6b1c4 100644 --- a/main/migrations/0029_fill_in_repo_data.py +++ b/main/migrations/0029_fill_in_repo_data.py @@ -7,7 +7,6 @@ from django.db import models class Migration(DataMigration): def forwards(self, orm): - "Write your forwards methods here." orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community') orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community') diff --git a/main/models.py b/main/models.py index a561f4f6..2f4d3520 100644 --- a/main/models.py +++ b/main/models.py @@ -7,10 +7,9 @@ from django.db import models from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field +from .utils import set_created_field from devel.models import DeveloperKey from packages.alpm import AlpmAPI @@ -140,7 +139,7 @@ class Package(models.Model): @property def signature(self): try: - data = b64decode(self.pgp_signature) + data = b64decode(self.pgp_signature.encode('utf-8')) except TypeError: return None if not data: @@ -187,7 +186,6 @@ class Package(models.Model): self._applicable_arches = list(arches) return self._applicable_arches - #@cache_function(119) def get_requiredby(self): """ Returns a list of package objects. An attempt will be made to keep this @@ -195,14 +193,22 @@ class Package(models.Model): category as this package if that check makes sense. """ from packages.models import Depend + sorttype = '''(CASE deptype + WHEN 'D' THEN 0 + WHEN 'O' THEN 1 + WHEN 'M' THEN 2 + WHEN 'C' THEN 3 + ELSE 1000 END)''' name_clause = '''packages_depend.name IN ( SELECT %s UNION ALL SELECT z.name FROM packages_provision z WHERE z.pkg_id = %s )''' requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').extra( - where=[name_clause], params=[self.pkgname, self.id]).order_by( - 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') + select={'sorttype': sorttype}, + where=[name_clause], params=[self.pkgname, self.id]).order_by( + 'sorttype', 'pkg__pkgname', + 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible requiredby = requiredby.filter( @@ -265,7 +271,6 @@ class Package(models.Model): trimmed.append(dep) return trimmed - #@cache_function(121) def get_depends(self): """ Returns a list of dicts. Each dict contains ('dep', 'pkg', and @@ -276,7 +281,6 @@ class Package(models.Model): Packages will match the testing status of this package if possible. """ deps = [] - arches = None # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): @@ -293,7 +297,6 @@ class Package(models.Model): return (sort_order.get(dep.deptype, 1000), dep.name) return sorted(deps, key=sort_key) - #@cache_function(123) def reverse_conflicts(self): """ Returns a list of packages with conflicts against this package. @@ -403,13 +406,13 @@ class Package(models.Model): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' names = [self.pkgname] - if self.pkgname.startswith('lib32-'): + if self.pkgname.startswith(u'lib32-'): names.append(self.pkgname[6:]) - elif self.pkgname.endswith('-multilib'): + elif self.pkgname.endswith(u'-multilib'): names.append(self.pkgname[:-9]) else: - names.append('lib32-' + self.pkgname) - names.append(self.pkgname + '-multilib') + names.append(u'lib32-' + self.pkgname) + names.append(self.pkgname + u'-multilib') return Package.objects.normal().filter( pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') diff --git a/main/utils.py b/main/utils.py index cdd4ff71..9ee8db58 100644 --- a/main/utils.py +++ b/main/utils.py @@ -3,11 +3,11 @@ try: except ImportError: import pickle -from datetime import datetime import hashlib from django.core.cache import cache from django.db import connections, router +from django.http import HttpResponse from django.utils.timezone import now from django.template.defaultfilters import slugify @@ -55,6 +55,14 @@ def clear_cache_function(func, args, kwargs): cache.delete(key) +def empty_response(): + empty = HttpResponse('') + # designating response as 'streaming' forces ConditionalGetMiddleware to + # not add a 'Content-Length: 0' header + empty.streaming = True + return empty + + def format_http_headers(request): headers = sorted((k, v) for k, v in request.META.items() if k.startswith('HTTP_')) diff --git a/mirrors/admin.py b/mirrors/admin.py index d6ea3950..9c88207d 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,3 @@ -import re from urlparse import urlparse, urlunsplit from django import forms @@ -36,22 +35,9 @@ class MirrorUrlInlineAdmin(admin.TabularInline): extra = 3 -# ripped off from django.forms.fields, adding netmask ability -IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') - -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) - - class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') class MirrorRsyncInlineAdmin(admin.TabularInline): diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..206c9d7d --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,49 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models +from south.modelsinspector import add_introspection_rules + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) + + +add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"]) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..b359b637 --- /dev/null +++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + if db.backend_name == 'postgres': + # For PostgreSQL, because it uses the 'inet' type and not a varchar + # column, we need to add an explict 'USING' cast to the SQL + # statement. We then execute the alter_column as well to ensure any + # of the other side-effects happen. + db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet') + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index e41f6b22..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,6 +6,7 @@ from django.db import models from django.db.models.signals import pre_save from django_countries import CountryField +from .fields import IPNetworkField from main.utils import set_created_field @@ -91,7 +92,7 @@ class MirrorUrl(models.Model): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False @@ -105,7 +106,7 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet - ip = models.CharField("IP", max_length=44) + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") created = models.DateTimeField(editable=False) @@ -136,6 +137,15 @@ class CheckLocation(models.Model): families = [x[0] for x in info] return families[0] + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 8ec85c40..241f5c61 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,13 +1,23 @@ -function mirror_status(chart_id, data_url) { - var jq_div = jQuery(chart_id); +function draw_graphs(location_url, log_url, container_id) { + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) + .then(function(loc_data, log_data) { + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); + }); + }); +} - var draw_graph = function(data) { +function mirror_status(container_id, check_loc, log_data, color) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); var margin = {top: 20, right: 20, bottom: 30, left: 40}, width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category10(), - x = d3.time.scale.utc().range([0, width]), + var x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); @@ -86,8 +96,9 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); var legend = svg.selectAll(".legend") - .data(color.domain()) + .data(active) .enter().append("g") .attr("class", "legend") .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); @@ -106,31 +117,52 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d; }); }; - /* invoke the data-fetch + first draw */ - var cached_data = null; - d3.json(data_url, function(json) { - cached_data = jQuery.map(json.urls, function(url, i) { + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } return { url: url.url, - logs: jQuery.map(url.logs, function(log, j) { - if (!log.is_success) { - return null; - } - return { - duration: log.duration, - check_time: new Date(log.check_time) - }; - }) + logs: logs }; }); - draw_graph(cached_data); - }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + jQuery(container_id).append('<h3><span class="fam-flag fam-flag-' + check_loc.country_code.toLowerCase() + '" title="' + check_loc.country + '"></span> ' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '</h3>'); + jQuery(container_id).append('<div id="' + chart_id + '" class="visualize-mirror visualize-chart"></div>'); + jQuery(container_id).append('<br/>'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - draw_graph(cached_data); + draw_graph('#' + chart_id, cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { diff --git a/mirrors/urls.py b/mirrors/urls.py index 4e929410..7cf76aa1 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), + (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 3ab176b3..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,22 +1,107 @@ from datetime import timedelta -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db import connection +from django.db.models import Count, Max, Min +from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): + +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + + +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + else: + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -24,63 +109,32 @@ def annotate_url(url, delays): else: # arbitrary small value divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() - if mirror_ids: - valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) - - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( - id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) - - vendor = database_vendor(MirrorUrl) - if vendor != 'sqlite': - url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) + url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.values_list( - 'url_id', 'check_time', 'last_sync').filter( - is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - if mirror_ids: - times = times.filter(url__mirror_id__in=mirror_ids) - delays = {} - for url_id, check_time, last_sync in times: - delay = check_time - last_sync - delays.setdefault(url_id, []).append(delay) - if urls: - url_data = dict((item['id'], item) for item in url_data) for url in urls: - for k, v in url_data.get(url.id, {}).items(): - if k not in ('id', 'mirror_id'): - setattr(url, k, v) - last_check = max([u.last_check for u in urls]) + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) - if mirror_ids: - check_info = check_info.filter(url__mirror_id__in=mirror_ids) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: @@ -93,12 +147,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): num_checks = 0 check_frequency = None - for url in urls: - # fake the standard deviation for local testing setups - if vendor == 'sqlite': - setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays) - return { 'cutoff': cutoff, 'last_check': last_check, @@ -108,8 +156,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): } -@cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -119,8 +166,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,11 +13,10 @@ from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -COUNTRY_LOOKUP = dict(COUNTRIES) - class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -28,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -45,7 +46,7 @@ class MirrorlistForm(forms.Form): country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): @@ -175,7 +176,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +194,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, @@ -264,7 +265,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): @@ -292,4 +294,32 @@ def status_json(request, tier=None): response = HttpResponse(to_json, content_type='application/json') return response + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all().order_by('pk') + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + # vim: set ts=4 sw=4 et: diff --git a/news/views.py b/news/views.py index 62d30fde..ca4fdf97 100644 --- a/news/views.py +++ b/news/views.py @@ -18,7 +18,7 @@ class NewsForm(forms.ModelForm): class NewsDetailView(DetailView): - model = News + queryset = News.objects.all().select_related('author') template_name = "news/view.html" diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ class Migration(DataMigration): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/models.py b/packages/models.py index 7bcdc000..f830aade 100644 --- a/packages/models.py +++ b/packages/models.py @@ -321,6 +321,16 @@ class Update(models.Model): return Package.objects.normal().filter( pkgname=self.pkgname, arch=self.arch) + def replacements(self): + pkgs = Package.objects.normal().filter( + replaces__name=self.pkgname) + if not self.arch.agnostic: + # make sure we match architectures if possible + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) + pkgs = pkgs.filter(arch__in=arches) + return pkgs + def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), self.pkgname, self.created) diff --git a/packages/sql/search_indexes.postgresql_psycopg2.sql b/packages/sql/search_indexes.postgresql_psycopg2.sql new file mode 100644 index 00000000..a7eaf998 --- /dev/null +++ b/packages/sql/search_indexes.postgresql_psycopg2.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX packages_pkgname_trgm_gist ON packages USING gist (UPPER(pkgname) gist_trgm_ops); +CREATE INDEX packages_pkgdesc_trgm_gist ON packages USING gist (UPPER(pkgdesc) gist_trgm_ops); diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..f7392a96 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -37,6 +37,12 @@ class BuildQueryStringNode(template.Node): def render(self, context): qs = parse_qs(context['current_query']) + # This is really dirty. The crazy round trips we do on our query string + # mean we get things like u'\xe2\x98\x83' in our views, when we should + # have simply u'\u2603' or a byte string of the UTF-8 value. Force the + # keys and list of values to be byte strings only. + qs = {k.encode('latin-1'): [v.encode('latin-1') for v in vals] + for k, vals in qs.items()} if 'sort' in qs and self.sortfield in qs['sort']: if self.sortfield.startswith('-'): qs['sort'] = [self.sortfield[1:]] @@ -53,10 +59,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ SELECT DISTINCT s.id """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 4c195385..c1f0f492 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -9,7 +9,7 @@ from django.db.models import Q from django.http import HttpResponse from django.shortcuts import redirect, render from django.views.decorators.cache import cache_control -from django.views.decorators.http import require_GET, require_POST +from django.views.decorators.http import require_safe, require_POST from main.models import Package, Arch from ..models import PackageRelation @@ -24,7 +24,7 @@ from .search import search_json from .signoff import signoffs, signoff_package, signoff_options, signoffs_json -@require_GET +@require_safe @cache_control(public=True, max_age=86400) def opensearch(request): if request.is_secure(): @@ -37,7 +37,7 @@ def opensearch(request): content_type='application/opensearchdescription+xml') -@require_GET +@require_safe @cache_control(public=True, max_age=300) def opensearch_suggest(request): search_term = request.GET.get('q', '') diff --git a/packages/views/display.py b/packages/views/display.py index fcf8fdea..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -7,6 +7,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from main.models import Package, PackageFile, Arch, Repo +from main.utils import empty_response from mirrors.utils import get_mirror_url_for_download from ..models import Update from ..utils import get_group_info, PackageJSONEncoder @@ -55,6 +56,8 @@ def recently_removed_package(request, name, repo, arch, cutoff=CUTOFF): try: update = match.latest() elsewhere = update.elsewhere() + if len(elsewhere) == 0: + elsewhere = update.replacements() if len(elsewhere) == 1: return redirect(elsewhere[0]) context = { @@ -124,6 +127,8 @@ def details(request, name='', repo='', arch=''): pkg = Package.objects.select_related( 'arch', 'repo', 'packager').get(pkgname=name, repo=repo_obj, arch=arch_obj) + if request.method == 'HEAD': + return empty_response() return render(request, 'packages/details.html', {'pkg': pkg}) except Package.DoesNotExist: # attempt a variety of fallback options before 404ing @@ -223,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 92b35b70..69ca3c9f 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def flag(request, name, repo, arch): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def flag(request, name, repo, arch): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { diff --git a/packages/views/search.py b/packages/views/search.py index 0362602e..b3778172 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -7,7 +7,7 @@ from django.http import HttpResponse from django.views.generic import ListView from main.models import Package, Arch, Repo -from main.utils import make_choice +from main.utils import empty_response, make_choice from ..models import PackageRelation from ..utils import attach_maintainers, PackageJSONEncoder @@ -99,6 +99,8 @@ class SearchListView(ListView): allowed_sort = list(sort_fields) + ["-" + s for s in sort_fields] def get(self, request, *args, **kwargs): + if request.method == 'HEAD': + return empty_response() self.form = PackageSearchForm(data=request.GET, show_staging=self.request.user.is_authenticated()) return super(SearchListView, self).get(request, *args, **kwargs) diff --git a/public/views.py b/public/views.py index 0dde88e7..71098c95 100644 --- a/public/views.py +++ b/public/views.py @@ -125,7 +125,6 @@ def keys(request): master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) - master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).order_by().values_list('signer').annotate( diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index c9f61964..f182cc33 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -20,7 +20,7 @@ class IsoListParser(HTMLParser): if tag == 'a': for name, value in attrs: if name == "href": - if value != '../' and self.url_re.search(value) != None: + if value != '../' and self.url_re.search(value) is not None: self.hyperlinks.append(value[:-1]) def parse(self, url): diff --git a/releng/models.py b/releng/models.py index b95f7d52..5ee2f325 100644 --- a/releng/models.py +++ b/releng/models.py @@ -160,7 +160,7 @@ class Release(models.Model): def torrent(self): try: - data = b64decode(self.torrent_data) + data = b64decode(self.torrent_data.encode('utf-8')) except TypeError: return None if not data: diff --git a/releng/views.py b/releng/views.py index ad4b07d1..b1c76a4a 100644 --- a/releng/views.py +++ b/releng/views.py @@ -231,7 +231,7 @@ def release_torrent(request, version): release = get_object_or_404(Release, version=version) if not release.torrent_data: raise Http404 - data = b64decode(release.torrent_data) + data = b64decode(release.torrent_data.encode('utf-8')) response = HttpResponse(data, content_type='application/x-bittorrent') # TODO: this is duplicated from Release.iso_url() filename = 'archlinux-%s-dual.iso.torrent' % release.version diff --git a/requirements.txt b/requirements.txt index 24f83d57..84450c76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.5 -Markdown==2.2.1 +Django==1.5.1 +IPy==0.81 +Markdown==2.3.1 South==0.7.6 bencode==1.0 django-countries==1.5 jsmin==2.0.2-1 pgpdump==1.4 -pytz>=2012j +pytz>=2013b diff --git a/requirements_prod.txt b/requirements_prod.txt index 0913d32a..cde5f513 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,12 +1,13 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.5 -Markdown==2.2.1 +Django==1.5.1 +IPy==0.81 +Markdown==2.3.1 South==0.7.6 bencode==1.0 django-countries==1.5 jsmin==2.0.2-1 pgpdump==1.4 -psycopg2==2.4.6 +psycopg2==2.5 pyinotify==0.9.4 python-memcached==1.48 -pytz>=2012j +pytz>=2013b diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html index 449731af..51cc8ba3 100644 --- a/retro/templates/retro/index-20030330.html +++ b/retro/templates/retro/index-20030330.html @@ -232,7 +232,6 @@ Happy holidays! <table width="100%"><tbody><tr><td align="right"> [ <a href="http://web.archive.org/web/20030330090147/http://www.archlinux.org/oldnews.php">Older News</a> ] </td></tr></tbody></table><br> -</p> <br> diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index a0dfb2ec..b7d6e1ee 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -314,7 +314,7 @@ dl { dl dt, dl dd { margin-bottom: 4px; - padding: 8px 0px 4px; + padding: 8px 0 4px; font-weight: bold; border-top: 1px dotted #bbb; } @@ -420,7 +420,7 @@ table thead th.tablesorter-headerDesc { } table thead th.sorter-false { - background-image: url(); + background-image: none; cursor: default; } @@ -490,13 +490,13 @@ table thead th.sorter-false { h3 span.arrow { display: block; - width: 0px; - height: 0px; + width: 0; + height: 0; border-left: 6px solid transparent; border-right: 6px solid transparent; border-top: 6px solid #1794D1; margin: 0 auto; - font-size: 0px; + font-size: 0; line-height: 0px; } @@ -885,19 +885,19 @@ table td.country { display: inline; } -#visualize-mirror .axis path, -#visualize-mirror .axis line { +.visualize-mirror .axis path, +.visualize-mirror .axis line { fill: none; stroke: #000; stroke-width: 3px; shape-rendering: crispEdges; } -#visualize-mirror .url-dot { +.visualize-mirror .url-dot { stroke: #000; } -#visualize-mirror .url-line { +.visualize-mirror .url-line { fill: none; stroke-width: 1.5px; } diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index 5809641b..f7be50d8 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -166,7 +166,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { (function($) { $.fn.enableCheckboxRangeSelection = function() { var lastCheckbox = null, - lastElement = null, spec = this; spec.unbind("click.checkboxrange"); diff --git a/templates/devel/clock.html b/templates/devel/clock.html index a7bc7315..8970fad0 100644 --- a/templates/devel/clock.html +++ b/templates/devel/clock.html @@ -56,6 +56,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/devel/index.html b/templates/devel/index.html index b2dfe077..f3f32863 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -294,7 +294,9 @@ </table> </div>{# #dash-by-developer #} {% endcache %} +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/devel/packages.html b/templates/devel/packages.html index 0aed22f4..ed96ad5a 100644 --- a/templates/devel/packages.html +++ b/templates/devel/packages.html @@ -73,6 +73,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/devel/profile.html b/templates/devel/profile.html index 16057275..dff5d925 100644 --- a/templates/devel/profile.html +++ b/templates/devel/profile.html @@ -24,7 +24,9 @@ </form> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 18f72125..98755ad2 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -106,11 +106,20 @@ {% endfor %} </tbody> </table> +</div> + +<div class="box"> + <h2>Mirror Status Charts</h2> - <h3>Mirror Status Chart</h3> + <p>Periodic checks of the mirrors are done from various geographic + locations, IP addresses, and using IPv4 or IPv6. These results are + summarized in graphical form below.</p> - <div id="visualize-mirror" class="visualize-chart"></div> + <div id="charts-container"></div> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "d3-3.0.6.min.js" %}"></script> <script type="text/javascript" src="{% static "archweb.js" %}"></script> @@ -122,7 +131,7 @@ $(document).ready(function() { headers: { 8: { sorter: 'mostlydigit' }, 9: { sorter: 'mostlydigit' }, 10: { sorter: 'mostlydigit' } } }); }); $(document).ready(function() { - mirror_status("#visualize-mirror", "./json/"); + draw_graphs("/mirrors/locations/json/", "./json/", "#charts-container"); }); </script> {% endblock %} diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 9050414d..de2c3d5c 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -87,20 +87,21 @@ </tr> </thead> <tbody> - {% for log in error_logs %} - {% spaceless %}<tr class="{% cycle 'odd' 'even' %}"> + {% for log in error_logs %}<tr class="{% cycle 'odd' 'even' %}"> <td>{{ log.url__url }}</td> <td>{{ log.url__protocol__protocol }}</td> <td class="country">{% country_flag log.country %}{{ log.country.name }}</td> <td class="wrap">{{ log.error|linebreaksbr }}</td> <td>{{ log.last_occurred|date:'Y-m-d H:i' }}</td> <td>{{ log.error_count }}</td> - </tr> - {% endspaceless %}{% endfor %} + </tr>{% endfor %} </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 2dd7ef49..e848a9c9 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -14,8 +14,7 @@ </tr> </thead> <tbody> - {% for m_url in urls %} - {% spaceless %}<tr class="{% cycle 'odd' 'even' %}"> + {% for m_url in urls %}<tr class="{% cycle 'odd' 'even' %}"> <td>{{ m_url.url }}</td> <td>{{ m_url.protocol }}</td> <td class="country">{% country_flag m_url.country %}{{ m_url.country.name }}</td> @@ -24,7 +23,6 @@ <td>{{ m_url.duration_avg|floatformat:2 }}</td> <td>{{ m_url.duration_stddev|floatformat:2 }}</td> <td>{{ m_url.score|floatformat:1|default:'∞' }}</td> - </tr> - {% endspaceless %}{% endfor %} + </tr>{% endfor %} </tbody> </table> diff --git a/templates/packages/details.html b/templates/packages/details.html index 36c422b2..07406fa2 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -229,7 +229,9 @@ </div> </div> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 4aa739c2..b89ffbfa 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -1,5 +1,4 @@ -{% load package_extras %}{% spaceless %}<li> -{% ifequal depend.pkg None %} +{% load package_extras %}<li>{% ifequal depend.pkg None %} {% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} <span class="virtual-dep">({% multi_pkg_details depend.providers %})</span> {% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} <span class="virtual-dep">(virtual)</span> {% endif %}{% else %} @@ -11,4 +10,4 @@ {% endif %}{% if depend.dep.deptype == 'M' %} <span class="make-dep"> (make)</span> {% endif %}{% if depend.dep.deptype == 'C' %} <span class="check-dep"> (check)</span> {% endif %}{% if depend.dep.description %} - <span class="dep-desc">{{ depend.dep.description }}</span> -{% endif %}</li>{% endspaceless %} +{% endif %}</li> diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index e8c713ac..504a322f 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -1,8 +1,8 @@ -{% load package_extras %}{% spaceless %}<li>{% pkg_details_link req.pkg %} +{% load package_extras %}<li>{% pkg_details_link req.pkg %} {% if req.name != pkg.pkgname %}<span class="virtual-dep"> (requires {{ req.name }})</span> {% endif %}{% if req.pkg.repo.testing %}<span class="testing-dep"> (testing)</span> {% endif %}{% if req.pkg.repo.staging %}<span class="staging-dep"> (staging)</span> {% endif %}{% if req.deptype == 'O' %}<span class="opt-dep"> (optional)</span> {% endif %}{% if req.deptype == 'M' %}<span class="make-dep"> (make)</span> {% endif %}{% if req.deptype == 'C' %}<span class="check-dep"> (check)</span> -{% endif %}</li>{% endspaceless %} +{% endif %}</li> diff --git a/templates/packages/differences.html b/templates/packages/differences.html index 38a94775..1a1bf260 100644 --- a/templates/packages/differences.html +++ b/templates/packages/differences.html @@ -5,7 +5,6 @@ {% block navbarclass %}anb-packages{% endblock %} {% block content %} -{% if differences %} <div class="box"> <h2>Package Differences by Architecture</h2> <div class="filter-criteria"> @@ -93,7 +92,9 @@ </table> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> @@ -109,5 +110,4 @@ $(document).ready(function() { filter_packages(); }); </script> -{% endif %} {% endblock %} diff --git a/templates/packages/groups.html b/templates/packages/groups.html index 95b1283c..c8f4aa5c 100644 --- a/templates/packages/groups.html +++ b/templates/packages/groups.html @@ -29,6 +29,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/packages/packages_list.html b/templates/packages/packages_list.html index f10f26be..110634b6 100644 --- a/templates/packages/packages_list.html +++ b/templates/packages/packages_list.html @@ -40,6 +40,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/packages/search.html b/templates/packages/search.html index 77e59813..b5b08268 100644 --- a/templates/packages/search.html +++ b/templates/packages/search.html @@ -69,8 +69,7 @@ </tr> </thead> <tbody> - {% for pkg in package_list %} - {% spaceless %}<tr class="{% cycle 'odd' 'even' %}"> + {% for pkg in package_list %}<tr class="{% cycle 'odd' 'even' %}"> {% if perms.main.change_package %} <td><input type="checkbox" name="pkgid" value="{{ pkg.id }}" /></td> {% endif %} @@ -85,8 +84,7 @@ <td class="wrap">{{ pkg.pkgdesc }}</td> <td>{{ pkg.last_update|date }}</td> <td>{{ pkg.flag_date|date }}</td> - </tr> - {% endspaceless %}{% endfor %} + </tr>{% endfor %} </tbody> </table> {% include "packages/search_paginator.html" %} diff --git a/templates/packages/signoffs.html b/templates/packages/signoffs.html index c3e75ae2..a159e998 100644 --- a/templates/packages/signoffs.html +++ b/templates/packages/signoffs.html @@ -50,9 +50,7 @@ </tr> </thead> <tbody id="tbody_signoffs"> - {% for group in signoff_groups %} - <tr class="{% cycle 'odd' 'even' %} {{ group.arch.name }} {{ group.target_repo|lower }}"> - {% spaceless %} + {% for group in signoff_groups %}<tr class="{% cycle 'odd' 'even' %} {{ group.arch.name }} {{ group.target_repo|lower }}"> <td>{% pkg_details_link group.package %} {{ group.version }}</td> <td>{{ group.arch.name }}</td> <td>{{ group.target_repo }}</td> @@ -75,12 +73,13 @@ {% endcomment %}{% if spec.known_bad %}Package is known to be bad<br/>{% endif %}{% comment %} {% endcomment %}{{ spec.comments|default:""|linebreaksbr }} {% endwith %}{% endif %}</td> - {% endspaceless %} - </tr> - {% endfor %} + </tr>{% endfor %} </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/packages/stale_relations.html b/templates/packages/stale_relations.html index c24715cd..233b2835 100644 --- a/templates/packages/stale_relations.html +++ b/templates/packages/stale_relations.html @@ -107,6 +107,9 @@ </form> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/public/keys.html b/templates/public/keys.html index c6da6c37..ab89423e 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -132,7 +132,9 @@ </tbody> </table> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "d3-3.0.6.min.js" %}"></script> <script type="text/javascript" src="{% static "archweb.js" %}"></script> diff --git a/templates/registration/login.html b/templates/registration/login.html index 226a0805..c722527d 100644 --- a/templates/registration/login.html +++ b/templates/registration/login.html @@ -21,7 +21,9 @@ </form> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/releng/iso_overview.html b/templates/releng/iso_overview.html index 196f0c0a..1a35ead4 100644 --- a/templates/releng/iso_overview.html +++ b/templates/releng/iso_overview.html @@ -36,6 +36,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/releng/release_list.html b/templates/releng/release_list.html index 5f248264..7197cc8d 100644 --- a/templates/releng/release_list.html +++ b/templates/releng/release_list.html @@ -39,7 +39,9 @@ </tbody> </table> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/releng/result_list.html b/templates/releng/result_list.html index 67e5934d..281df083 100644 --- a/templates/releng/result_list.html +++ b/templates/releng/result_list.html @@ -33,6 +33,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/todolists/list.html b/templates/todolists/list.html index b7d10264..d950db94 100644 --- a/templates/todolists/list.html +++ b/templates/todolists/list.html @@ -46,6 +46,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/todolists/view.html b/templates/todolists/view.html index 644e78b0..e0cd4007 100644 --- a/templates/todolists/view.html +++ b/templates/todolists/view.html @@ -100,6 +100,9 @@ </tbody> </table> </div> +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} <script type="text/javascript" src="{% static "archweb.js" %}"></script> <script type="text/javascript"> diff --git a/templates/visualize/index.html b/templates/visualize/index.html index bd98dcae..e091c761 100644 --- a/templates/visualize/index.html +++ b/templates/visualize/index.html @@ -23,7 +23,9 @@ </div> <div id="visualize-archrepo" class="visualize-chart"></div> </div> +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %} <script type="text/javascript" src="{% static "d3-3.0.6.min.js" %}"></script> <script type="text/javascript" src="{% static "archweb.js" %}"></script> diff --git a/todolists/utils.py b/todolists/utils.py index 51a75a3c..7b98c887 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,5 +1,4 @@ from django.db import connections, router -from django.db.models import Count from .models import Todolist, TodolistPackage from packages.models import Package diff --git a/todolists/views.py b/todolists/views.py index 7de32036..b6d3c25f 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -10,7 +10,6 @@ from django.db import transaction from django.views.decorators.cache import never_cache from django.views.generic import DeleteView from django.template import Context, loader -from django.template.defaultfilters import slugify from django.utils.timezone import now from main.models import Package, Repo diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js index 7e240d44..5004fe6c 100644 --- a/visualize/static/visualize.js +++ b/visualize/static/visualize.js @@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) { var nodes = d3_div.data([json]).selectAll("div") .data(treemap.nodes, key_func); /* start out new nodes in the center of the picture area */ - var w_center = jq_div.width() / 2; + var w_center = jq_div.width() / 2, h_center = jq_div.height() / 2; nodes.enter().append("div") .attr("class", "treemap-cell") |