Add pagination to JSON package search (#73)release_2018-01-02

* Add pagination to JSON search form using Paginator

The downside of this is that we can't easily run prefetch_related
anymore, but given the limit I don't suspect this to be a problem.

Signed-off-by: Johannes Löthberg <johannes@kyriasis.com>

* packages/views/search: Add appropriate newlines to search_json

Signed-off-by: Johannes Löthberg <johannes@kyriasis.com>

* Update package fixture to be the same as devel/fixtures/core.db.tar.gz

Signed-off-by: Johannes Löthberg <johannes@kyriasis.com>

* packages/tests: Add tests for JSON search pagination

Signed-off-by: Johannes Löthberg <johannes@kyriasis.com>

3 files changed, 176 insertions, 32 deletions

diff --git a/main/fixtures/package.json b/main/fixtures/package.json
index 2ed00389..d191027a 100644
--- a/main/fixtures/package.json
+++ b/main/fixtures/package.json
@@ -1,25 +1,132 @@
 [
-    {
-        "fields": {
-            "arch": 2,
-            "build_date": "2017-04-27T10:37:12Z",
-            "compressed_size": 61650988,
-            "created": "2017-05-20T13:06:46.688Z",
-            "epoch": 0,
-            "filename": "linux-4.10.13-1-x86_64.pkg.tar.xz",
-            "installed_size": 74946560,
-            "last_update": "2017-05-20T13:06:46.688Z",
-            "packager_str": "Tobias Powalowski <tpowa@archlinux.org>",
-            "pkgbase": "linux",
-            "pkgdesc": "The Linux kernel and modules",
-            "pkgname": "linux",
-            "pkgrel": "1",
-            "pkgver": "4.10.13",
-            "repo": 1,
-            "signature_bytes": "iQEzBAABCAAdFiEEW34/txt/EDKaHAOrdx32Yn7faB8FAlkC3M8ACgkQdx32Yn7faB8kHAf/YSMTEnIyLK/dTFWjM9P/X8iUVzoJUVn2X76m5QrMVLdX8rrqAXZt74DmEl87X4cjEweHgM3ihhP2L1i5YcFZ2t1NFhApJcdElPRZpLP/0BjR7ZoY9aUfnpseyBzzN+LamkUjAMdsQRBJDP3URfIFf+/r+F2bda0yto7h9yekuOphltkISF6ABn3VPrO5AxgI6SNrsXRdek3AUuOWr8BSAuIeRYt2chDuRPmztAy7DaDcZ71V9S90F+u2FkpcLKJygAAQbfKIPHVTS5GL2wc8gO5jXw+8lj4ioj4/fE+/Nq/ytn0zQHdebQ9akKzW+13D2r49pEU8EZiianmOgwd6yw==",
-            "url": "https://www.kernel.org/"
-        },
-        "model": "main.package",
-        "pk": 1
-    }
+  {
+    "fields": {
+      "build_date": "2017-06-24T07:08:48Z",
+      "compressed_size": 64291412,
+      "pkgname": "linux",
+      "files_last_update": null,
+      "pkgdesc": "The Linux kernel and modules",
+      "created": "2017-12-30T02:21:43.788Z",
+      "url": "https://www.kernel.org/",
+      "epoch": 0,
+      "filename": "linux-4.11.7-1-x86_64.pkg.tar.xz",
+      "repo": 1,
+      "signature_bytes": "iQEzBAABCAAdFiEEW34/txt/EDKaHAOrdx32Yn7faB8FAllOGpcACgkQdx32Yn7faB9xnAf/fzWz4rAEWx97ad2DAD3kzy7yZK/sCx4lC762kyap2INJWJYwu7K46ub+NE7K7YlORg2wOn8H8Do5X40Q6/NWLS7f6YZqdL3PyIkdjB+oyblOsYR+DsWWY67lyhfT1FDJyqrdfAYbMptgubWvyjPcMeOJ2DwK2wa9T4mnSenXEOx/kbGhjIB+zWjEHvnW/Sptp5LYjSYKfBOqj60Y4pZPfoBl5POl47PRwMgFerA3UTqtylYlrnx3BmeNbi5IUmQYUmbpl0DxvX02zew11sP+f8y9P4hEnJgQazvl6V/q7ekbSScB/NtLr3vyCTkprKX9Un0+B224B0jY+twH3pLkFw==",
+      "pkgbase": "linux",
+      "pkgrel": "1",
+      "flag_date": null,
+      "installed_size": 84407296,
+      "packager_str": "Tobias Powalowski <tpowa@archlinux.org>",
+      "last_update": "2017-12-30T02:21:43.788Z",
+      "arch": 3,
+      "packager": null,
+      "pkgver": "4.11.7"
+    },
+    "model": "main.package",
+    "pk": 1
+  },
+  {
+    "fields": {
+      "build_date": "2017-03-12T14:09:38Z",
+      "compressed_size": 2288568,
+      "pkgname": "coreutils",
+      "files_last_update": null,
+      "pkgdesc": "The basic file, shell and text manipulation utilities of the GNU operating system",
+      "created": "2017-12-30T02:21:43.808Z",
+      "url": "https://www.gnu.org/software/coreutils/",
+      "epoch": 0,
+      "filename": "coreutils-8.27-1-x86_64.pkg.tar.xz",
+      "repo": 1,
+      "signature_bytes": "iQIrBAABCAAdFiEEdPGyFg//myr+77hJPByHYDC2X+IFAljFVskACgkQPByHYDC2X+J+Wg/AjTXNEb9xwV84Se8XzIowoTjUaoFAar9+VfOBGarQbmOwyU6I+jMtSXI4VTTrNBJzfarA3GEXEUzBa7OVRgB04yU5puAdnRHWIBU5Ma6fkmrW/JUR/sF1A+mhJncZvs/D19TucsEKCHuyqguOMmJN1YTwRtsbE4qbj+Kc4QZXYaH08I5qQXQds6wf0eIc1D6qV2ZNivbm5LusSUZ8UnO3kSJf6gMKGxEy+JKckSS/Q0VgHp8TXNAodOOF0V81fxdu/apye3JgWRJgnP+V+kSOSG11TODBVZlW51U2+ADuSroDNW2Lws52eNHVblyNOlIY71jlk++KsfLopqUGk62ZLt+JMZO75XSbrJF8ZPvLwa4xx9iq1Z8ywoNxVGM3wKhPeeYebJbr1QLctcrQdzznNKbUCR48US+FmQBVqx5Xov9NT4ZuoDazPDefLfMChX/oEDSUZUg3dC9ffjyJG3bcWGO1TJb7iK9ZzEuhR9SHE8r40qpaFxZbZ+nur7gkMmDiTB6Xf27zZmcw8cNf7mg+m2gYeMFA9Ivg2HRkK/MtYPBj3ljjgBLPVOzEJPrIjxz04CiesX3JDYNAh6qN+8gQh0B90Jj3dHnXJRlQolMCgnzQQQy8kGjC0nZCPK04dGpMGYpNKLG4f9CqwhJDKEUvtYh/HNF9HTSj",
+      "pkgbase": "coreutils",
+      "pkgrel": "1",
+      "flag_date": null,
+      "installed_size": 14427136,
+      "packager_str": "Sébastien Luttringer <seblu@seblu.net>",
+      "last_update": "2017-12-30T02:21:43.808Z",
+      "arch": 3,
+      "packager": null,
+      "pkgver": "8.27"
+    },
+    "model": "main.package",
+    "pk": 2
+  },
+  {
+    "fields": {
+      "build_date": "2017-06-21T09:00:35Z",
+      "compressed_size": 8850512,
+      "pkgname": "glibc",
+      "files_last_update": null,
+      "pkgdesc": "GNU C Library",
+      "created": "2017-12-30T02:21:43.825Z",
+      "url": "http://www.gnu.org/software/libc",
+      "epoch": 0,
+      "filename": "glibc-2.25-5-x86_64.pkg.tar.xz",
+      "repo": 1,
+      "signature_bytes": "iQEzBAABCAAdFiEEghj4iEmqxSLpTPRwpekojE+kFfoFAllKSO8ACgkQpekojE+kFfrjjwgAmqv4C9UY2inHokoSZAL49ic3K5IjAG209DckVbOvOTIKljvqGrJb5G47fQ8y5OfK5J39R98AIyq57oMgcUwfr1TZBnKOHiKOJDwurGGj9+PP3XSGIYpQxt4YBEt9Sd30J3KTCqS/bHyO/y3Il3Cg5k/5JNqu6sPqrheqjPwVXpvCbXfQUK5IKx45j8qmO5j/HZtqyuB497DkmZ3gDSn63YqoE5A51Ap8cMRp3hCdplgVh1T6T2wp/DK5PmNWqxrV0Qgd8Xii9yigMCSsQRR+ivEJI6Hh3gYw8eyT/L+AR2VB45u3jywzH+wan4ic5qjEl9yX0C+/nwf3w37FVm7pNA==",
+      "pkgbase": "glibc",
+      "pkgrel": "5",
+      "flag_date": null,
+      "installed_size": 38140928,
+      "packager_str": "Jan Alexander Steffens (heftig) <jan.steffens@gmail.com>",
+      "last_update": "2017-12-30T02:21:43.825Z",
+      "arch": 3,
+      "packager": null,
+      "pkgver": "2.25"
+    },
+    "model": "main.package",
+    "pk": 3
+  },
+  {
+    "fields": {
+      "build_date": "2017-06-13T12:11:40Z",
+      "compressed_size": 753348,
+      "pkgname": "pacman",
+      "files_last_update": null,
+      "pkgdesc": "A library-based package manager with dependency support",
+      "created": "2017-12-30T02:21:43.848Z",
+      "url": "http://www.archlinux.org/pacman/",
+      "epoch": 0,
+      "filename": "pacman-5.0.2-1-x86_64.pkg.tar.xz",
+      "repo": 1,
+      "signature_bytes": "iQFCBAABCAAsFiEEAv0cepNOYUVFhJ8ZpiNAdEmOnO4FAlk/1rAOHGFyY2hAZXdvcm0uZGUACgkQpiNAdEmOnO5kXAf/bfbuoh1VNU/tOjfAAaR8ahtxihvqobMfpLTIeLuZDjk0U1sEE8ltk7Ky8wwFWduz1SSjeCuSm/020zizNXrCv1Wq3x6mOANs2VY+JDbWMk48KAR9dajY1buu8rWRq+Rt0mFA8XTZhVXhijG7hXYWSVBtcokevrN3cDizxchb6gQeh0xYE7/vlHwblch+Y+MXMR0bbjF8xEIITZYr4895i8xvuJtD6Ll7Wr/WBGXdFlm9Z0LmAZlDHDusJ7cC5gr7HPhUcKg4tnm8mtVmCJugoiHUVel5llZ1MW4iUCcx4suUyQAqKa9HqxXE9e5ADzZHBdZA4hJ6UQQwDRm57cLwFQ==",
+      "pkgbase": "pacman",
+      "pkgrel": "1",
+      "flag_date": null,
+      "installed_size": 4694016,
+      "packager_str": "Christian Hesse <arch@eworm.de>",
+      "last_update": "2017-12-30T02:21:43.848Z",
+      "arch": 3,
+      "packager": null,
+      "pkgver": "5.0.2"
+    },
+    "model": "main.package",
+    "pk": 4
+  },
+  {
+    "fields": {
+      "build_date": "2017-01-28T22:32:41Z",
+      "compressed_size": 3905996,
+      "pkgname": "systemd",
+      "files_last_update": null,
+      "pkgdesc": "system and service manager",
+      "created": "2017-12-30T02:21:43.865Z",
+      "url": "https://www.github.com/systemd/systemd",
+      "epoch": 0,
+      "filename": "systemd-232-8-x86_64.pkg.tar.xz",
+      "repo": 1,
+      "signature_bytes": "iQEzBAABCAAdFiEESH6swIVXrQggiNq6HrJjj/VsDFMFAliOFSIACgkQHrJjj/VsDFPKYgf/aM3TnZOHoU4avTZt0KmPszRN9o4iQD50buvwgre3kt7WiiOqxpPgyML7B/s0erT3bgz22JD199kC6RbTn+qBqo/fBPZOywX0NhVIqgPsHvZjxuDJeehboqlePtJ4LtLG882TQxb1bsf8PBzTqmX7H8n/2lEommWwiruVo9bwOeaCh5ax9V8/a6UaEMkD4VwLUkAOQkVLvi8HZG7YbczotSMhhU9tSyH36UCUaA/xW3qiibD4WT2lZbKnbaC3JtddEfJn0hrwOqkiVjN0fDWnO3PLw/KOprZ18q4Be7j8AyYBdEUtEdZEmJUkqzaAKFp+3EzL1bq+g7S8nlf/Q929bA==",
+      "pkgbase": "systemd",
+      "pkgrel": "8",
+      "flag_date": null,
+      "installed_size": 18100224,
+      "packager_str": "Dave Reisner <dreisner@archlinux.org>",
+      "last_update": "2017-12-30T02:21:43.865Z",
+      "arch": 3,
+      "packager": null,
+      "pkgver": "232"
+    },
+    "model": "main.package",
+    "pk": 5
+  }
 ]
diff --git a/packages/tests.py b/packages/tests.py
index 69bf2e76..5c70aaf4 100644
--- a/packages/tests.py
+++ b/packages/tests.py
@@ -75,8 +75,9 @@ class PackageSearchJson(TestCase):
         response = self.client.get('/packages/search/json/?repository=core')
         self.assertEqual(response.status_code, 200)
         data = json.loads(response.content)
-        self.assertEqual(len(data['results']), 1)
-        self.assertEqual(data['results'][0]['pkgname'], 'linux')
+        self.assertEqual(len(data['results']), 5)
+        self.assertEqual(set(map(lambda r: r['pkgname'], data['results'])),
+                         {"coreutils", "glibc", "linux", "pacman", "systemd"})
 
     def test_packagename(self):
         response = self.client.get('/packages/search/json/?name=linux')
@@ -90,6 +91,23 @@ class PackageSearchJson(TestCase):
         data = json.loads(response.content)
         self.assertEqual(len(data['results']), 0)
 
+    def test_limit_four(self):
+        response = self.client.get('/packages/search/json/?limit=4')
+        self.assertEqual(response.status_code, 200)
+        data = json.loads(response.content)
+        self.assertEqual(data['page'], 1)
+        self.assertEqual(data['num_pages'], 2)
+        self.assertEqual(data['limit'], 4)
+        self.assertEqual(len(data['results']), 4)
+
+    def test_second_page(self):
+        response = self.client.get('/packages/search/json/?limit=4&page=2')
+        self.assertEqual(response.status_code, 200)
+        data = json.loads(response.content)
+        self.assertEqual(data['page'], 2)
+        self.assertEqual(data['num_pages'], 2)
+        self.assertEqual(len(data['results']), 1)
+
 
 class PackageSearch(TestCase):
     fixtures = ['main/fixtures/arches.json', 'main/fixtures/repos.json',
@@ -113,7 +131,7 @@ class PackageSearch(TestCase):
     def test_filter_repo(self):
         response = self.client.get('/packages/?repo=Core')
         self.assertEqual(response.status_code, 200)
-        self.assertIn('1 matching package found', response.content)
+        self.assertIn('5 matching packages found', response.content)
 
     def test_filter_desc(self):
         response = self.client.get('/packages/?desc=kernel')
@@ -128,7 +146,7 @@ class PackageSearch(TestCase):
     def test_filter_not_flagged(self):
         response = self.client.get('/packages/?flagged=Not Flagged')
         self.assertEqual(response.status_code, 200)
-        self.assertIn('1 matching package found', response.content)
+        self.assertIn('5 matching packages found', response.content)
 
     def test_filter_arch(self):
         response = self.client.get('/packages/?arch=any')
@@ -138,11 +156,11 @@ class PackageSearch(TestCase):
     def test_filter_maintainer_orphan(self):
         response = self.client.get('/packages/?maintainer=orphan')
         self.assertEqual(response.status_code, 200)
-        self.assertIn('1 matching package found', response.content)
+        self.assertIn('5 matching packages found', response.content)
 
     def test_filter_packager_unknown(self):
         response = self.client.get('/packages/?packager=unknown')
         self.assertEqual(response.status_code, 200)
-        self.assertIn('1 matching package found', response.content)
+        self.assertIn('5 matching packages found', response.content)
 
 # vim: set ts=4 sw=4 et:
diff --git a/packages/views/search.py b/packages/views/search.py
index 6e892251..264e7bf6 100644
--- a/packages/views/search.py
+++ b/packages/views/search.py
@@ -1,6 +1,7 @@
 import json
 
 from django import forms
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.contrib.auth.models import User
 from django.db.models import Q
 from django.http import HttpResponse
@@ -14,6 +15,8 @@ from ..utils import attach_maintainers, PackageJSONEncoder
 
 
 class PackageSearchForm(forms.Form):
+    limit = forms.CharField(required=False)
+    page = forms.CharField(required=False)
     repo = forms.MultipleChoiceField(required=False)
     arch = forms.MultipleChoiceField(required=False)
     name = forms.CharField(required=False)
@@ -155,13 +158,29 @@ def search_json(request):
         form = PackageSearchForm(data=request.GET,
                 show_staging=request.user.is_authenticated())
         if form.is_valid():
+            form_limit = form.cleaned_data['limit']
+            limit = min(limit, int(form_limit)) if form_limit else limit
+            container['limit'] = limit
+
             packages = Package.objects.select_related('arch', 'repo',
                     'packager')
             if not request.user.is_authenticated():
                 packages = packages.filter(repo__staging=False)
-            packages = parse_form(form, packages)[:limit]
-            packages = packages.prefetch_related('groups', 'licenses',
-                    'conflicts', 'provides', 'replaces', 'depends')
+            packages = parse_form(form, packages)
+
+            paginator = Paginator(packages, limit)
+            container['num_pages'] = paginator.num_pages
+
+            page = form.cleaned_data.get('page')
+            page = int(page) if page else 1
+            container['page'] = page
+            try:
+                packages = paginator.page(page)
+            except PageNotAnInteger:
+                packages = paginator.page(1)
+            except EmptyPage:
+                packages = paginator.page(paginator.num_pages)
+
             attach_maintainers(packages)
             container['results'] = packages
             container['valid'] = True