diff options
author | jelle van der Waa <jelle@vdwaa.nl> | 2018-07-23 22:23:15 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-07-23 22:23:15 +0200 |
commit | 559b8bc7fb6593f9d550f9b9c18c3b9a06743f27 (patch) | |
tree | 861db0e3b7a38175b0ec1e91182c0b6695c7c62d | |
parent | 196b1fd8950abe9137e1583f172090b888d487ef (diff) | |
parent | d1abef2064cb293b2418726ea82b2c0c7c107b7f (diff) |
Merge pull request #129 from jelly/mirrorlog_cleanuprelease_2018-07-25
Mirrorlog cleanup
-rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 13 | ||||
-rw-r--r-- | mirrors/models.py | 3 | ||||
-rw-r--r-- | mirrors/tests/__init__.py | 2 | ||||
-rw-r--r-- | mirrors/tests/test_mirrorcheck.py | 58 | ||||
-rw-r--r-- | mirrors/tests/test_mirrorresolv.py | 44 | ||||
-rw-r--r-- | settings.py | 3 |
6 files changed, 118 insertions, 5 deletions
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 9eaf38f1..c78ad7b7 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,7 @@ Usage: ./manage.py mirrorcheck """ from collections import deque -from datetime import datetime +from datetime import datetime, timedelta from httplib import HTTPException import logging import os @@ -29,6 +29,7 @@ import urllib2 from django.core.management.base import BaseCommand from django.db import transaction +from django.conf import settings from django.utils.timezone import now from mirrors.models import MirrorUrl, MirrorLog, CheckLocation @@ -84,6 +85,7 @@ class Command(BaseCommand): pool = MirrorCheckPool(urls, location, timeout) pool.run() + pool.cleanup() return 0 @@ -246,7 +248,7 @@ class MirrorCheckPool(object): def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() - for url in list(urls): + for url in urls: self.tasks.put(url) self.threads = [] for _ in range(num_threads): @@ -266,4 +268,11 @@ class MirrorCheckPool(object): MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") + def cleanup(self): + days = getattr(settings, 'MIRRORLOG_RETENTION_PERIOD', 365) + removal_date = now() - timedelta(days=days) + logger.info("cleaning up older MirrorLog objects then %s", removal_date.strftime('%Y-%m-%d')) + MirrorLog.objects.filter(check_time__lt=removal_date).delete() + logger.info('Finished cleaning up old MirrorLog objects') + # vim: set ts=4 sw=4 et: diff --git a/mirrors/models.py b/mirrors/models.py index 795d0294..35347e7d 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -150,8 +150,7 @@ class CheckLocation(models.Model): @property def family(self): - info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, - socket.AI_NUMERICHOST) + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, socket.AI_NUMERICHOST) families = [x[0] for x in info] return families[0] diff --git a/mirrors/tests/__init__.py b/mirrors/tests/__init__.py index a1d3c2c9..4ea5e1fb 100644 --- a/mirrors/tests/__init__.py +++ b/mirrors/tests/__init__.py @@ -2,7 +2,7 @@ from mirrors.models import MirrorUrl, MirrorProtocol, Mirror def create_mirror_url(name='mirror1', country='US', - protocol='http', url='https://archlinux.org'): + protocol='http', url='https://archlinux.org/'): mirror = Mirror.objects.create(name=name, admin_email='admin@archlinux.org') mirror_protocol = MirrorProtocol.objects.create(protocol=protocol) diff --git a/mirrors/tests/test_mirrorcheck.py b/mirrors/tests/test_mirrorcheck.py new file mode 100644 index 00000000..1f22d1c5 --- /dev/null +++ b/mirrors/tests/test_mirrorcheck.py @@ -0,0 +1,58 @@ +import mock +import time + + +from django.utils.timezone import now +from datetime import timedelta + + +from django.test import TestCase +from django.core.management import call_command + + +from mirrors.tests import create_mirror_url +from mirrors.models import MirrorLog + + +class MirrorCheckTest(TestCase): + def setUp(self): + self.mirror_url = create_mirror_url() + + def tearDown(self): + self.mirror_url.delete() + + @mock.patch('urllib2.Request') + @mock.patch('urllib2.urlopen') + def test_invalid(self, urlopen, Request): + urlopen.return_value.read.return_value = 'data' + Request.get_host.return_value = 'archlinux.org' + Request.type.return_value = 'https' + + call_command('mirrorcheck') + mirrorlog = MirrorLog.objects.first() + self.assertNotEqual(mirrorlog.error, '') + self.assertEqual(mirrorlog.is_success, False) + + @mock.patch('urllib2.Request') + @mock.patch('urllib2.urlopen') + def test_valid(self, urlopen, Request): + urlopen.return_value.read.return_value = str(int(time.time())) + Request.get_host.return_value = 'archlinux.org' + Request.type.return_value = 'https' + + call_command('mirrorcheck') + mirrorlog = MirrorLog.objects.first() + self.assertEqual(mirrorlog.error, '') + self.assertEqual(mirrorlog.is_success, True) + + @mock.patch('urllib2.Request') + @mock.patch('urllib2.urlopen') + def test_valid(self, urlopen, Request): + urlopen.return_value.read.return_value = str(int(time.time())) + Request.get_host.return_value = 'archlinux.org' + Request.type.return_value = 'https' + + date = now() - timedelta(days=600) + MirrorLog.objects.create(url=self.mirror_url, check_time=date) + call_command('mirrorcheck') + self.assertEqual(len(MirrorLog.objects.all()), 1) diff --git a/mirrors/tests/test_mirrorresolv.py b/mirrors/tests/test_mirrorresolv.py new file mode 100644 index 00000000..8b37e764 --- /dev/null +++ b/mirrors/tests/test_mirrorresolv.py @@ -0,0 +1,44 @@ +import mock + +from django.test import TestCase +from django.core.management import call_command + +from mirrors.tests import create_mirror_url + + +class MirrorCheckTest(TestCase): + def setUp(self): + self.mirror_url = create_mirror_url() + + def tearDown(self): + self.mirror_url.delete() + + @mock.patch('socket.getaddrinfo') + def test_ip4_ip6(self, getaddrinfo): + getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))] + call_command('mirrorresolv') + self.mirror_url.refresh_from_db() + self.assertEqual(self.mirror_url.has_ipv4, True) + self.assertEqual(self.mirror_url.has_ipv6, True) + + @mock.patch('socket.getaddrinfo') + def test_ip4_only(self, getaddrinfo): + getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0))] + call_command('mirrorresolv') + self.mirror_url.refresh_from_db() + self.assertEqual(self.mirror_url.has_ipv4, True) + self.assertEqual(self.mirror_url.has_ipv6, False) + + @mock.patch('socket.getaddrinfo') + def test_running_twice(self, getaddrinfo): + getaddrinfo.return_value = [(2, 1, 6, '', ('1.1.1.1', 0)), (10, 1, 6, '', ('1a01:3f8:132:1d96::1', 0, 0, 0))] + + # Check if values changed + with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger: + call_command('mirrorresolv', '-v3') + self.assertEqual(logger.debug.call_count, 4) + + # running again does not change any values. + with mock.patch('mirrors.management.commands.mirrorresolv.logger') as logger: + call_command('mirrorresolv', '-v3') + self.assertEqual(logger.debug.call_count, 3) diff --git a/settings.py b/settings.py index df447b3e..6022d572 100644 --- a/settings.py +++ b/settings.py @@ -145,6 +145,9 @@ TORRENT_TRACKERS = ( 'http://tracker.archlinux.org:6969/announce', ) +# How long to keep mirrorlog's in days +MIRRORLOG_RETENTION_PERIOD = 365 + # Shorten some names just a bit COUNTRIES_OVERRIDE = { 'GB': 'United Kingdom', |