Automate mirror submission

.gitignore

@@ -6,6 +6,7 @@ local_settings.py
 archweb.db
 archweb.db-*
 database.db
+/*.tar.gz
 tags
 collected_static/
 testing/

mirrors/urls_mirrorlist.py

@@ -3,6 +3,7 @@
 
 urlpatterns = [
     path('', views.generate_mirrorlist, name='mirrorlist'),
+    path('submit/', views.submit_mirror, name='mirrorsubmit'),
     re_path(r'^all/$', views.find_mirrors, {'countries': ['all']}),
     re_path(r'^all/(?P<protocol>[A-z]+)/$', views.find_mirrors_simple, name='mirrorlist_simple')
 ]

mirrors/views/mirrorlist.py

@@ -1,17 +1,132 @@
 from operator import attrgetter, itemgetter
+from urllib.parse import urlparse, urlunsplit
+from functools import partial
 
 from django import forms
+from django.db import DatabaseError, transaction
 from django.db.models import Q
-from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
+from django.core.mail import send_mail
+from django.template import loader
+from django.forms.widgets import (
+    Select,
+    SelectMultiple,
+    CheckboxSelectMultiple,
+    TextInput,
+    EmailInput
+)
 from django.shortcuts import get_object_or_404, redirect, render
 from django.views.decorators.csrf import csrf_exempt
 from django_countries import countries
 
-from ..models import MirrorUrl, MirrorProtocol
+from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync
 from ..utils import get_mirror_statuses
 
 import random
 
+# This is populated later, and re-populated every refresh
+# This was the only way to get 3 different examples without
+# changing the models.py
+url_examples = []
+
+
+class MirrorRequestForm(forms.ModelForm):
+    upstream = forms.ModelChoiceField(
+        queryset=Mirror.objects.filter(
+            tier__gte=0,
+            tier__lte=1
+        ),
+        required=False
+    )
+
+    class Meta:
+        model = Mirror
+        fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email',
+                  'isos', 'active', 'public', 'rsync_user', 'rsync_password', 'notes')
+
+    def __init__(self, *args, **kwargs):
+        super(MirrorRequestForm, self).__init__(*args, **kwargs)
+        fields = self.fields
+        fields['name'].widget.attrs.update({'placeholder': 'Ex: mirror.argentina.co'})
+        fields['rsync_user'].widget.attrs.update({'placeholder': 'Optional'})
+        fields['rsync_password'].widget.attrs.update({'placeholder': 'Optional'})
+        fields['notes'].widget.attrs.update({'placeholder': 'Ex: Hosted by ISP GreatISO.bg'})
+
+    def as_div(self):
+        "Returns this form rendered as HTML <divs>s."
+        return self._html_output(
+            normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+            error_row=u'%s',
+            row_ender='</div>',
+            help_text_html=u' <span class="helptext">%s</span>',
+            errors_on_separate_row=True)
+
+
+class MirrorUrlForm(forms.ModelForm):
+    class Meta:
+        model = MirrorUrl
+        fields = ('url', 'country', 'bandwidth', 'active')
+
+    def __init__(self, *args, **kwargs):
+        global url_examples
+
+        super(MirrorUrlForm, self).__init__(*args, **kwargs)
+        fields = self.fields
+
+        if len(url_examples) == 0:
+            url_examples = [
+                'Ex: http://mirror.argentina.co/archlinux',
+                'Ex: https://mirror.argentina.co/archlinux',
+                'Ex: rsync://mirror.argentina.co/archlinux'
+            ]
+
+        fields['url'].widget.attrs.update({'placeholder': url_examples.pop()})
+
+    def clean_url(self):
+        # is this a valid-looking URL?
+        url_parts = urlparse(self.cleaned_data["url"])
+        if not url_parts.scheme:
+            raise forms.ValidationError("No URL scheme (protocol) provided.")
+        if not url_parts.netloc:
+            raise forms.ValidationError("No URL host provided.")
+        if url_parts.params or url_parts.query or url_parts.fragment:
+            raise forms.ValidationError(
+                "URL parameters, query, and fragment elements are not supported.")
+        # ensure we always save the URL with a trailing slash
+        path = url_parts.path
+        if not path.endswith('/'):
+            path += '/'
+        url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', ''))
+        return url
+
+    def as_div(self):
+        "Returns this form rendered as HTML <divs>s."
+        return self._html_output(
+            normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+            error_row=u'%s',
+            row_ender='</div>',
+            help_text_html=u' <span class="helptext">%s</span>',
+            errors_on_separate_row=True)
+
+
+class MirrorRsyncForm(forms.ModelForm):
+    class Meta:
+        model = MirrorRsync
+        fields = ('ip',)
+
+    def __init__(self, *args, **kwargs):
+        super(MirrorRsyncForm, self).__init__(*args, **kwargs)
+        fields = self.fields
+        fields['ip'].widget.attrs.update({'placeholder': 'Ex: 1.2.4.5'})
+
+    def as_div(self):
+        "Returns this form rendered as HTML <divs>s."
+        return self._html_output(
+            normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+            error_row=u'%s',
+            row_ender='</div>',
+            help_text_html=u' <span class="helptext">%s</span>',
+            errors_on_separate_row=True)
+
 
 class MirrorlistForm(forms.Form):
     country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'}))
@@ -127,4 +242,92 @@ def find_mirrors_simple(request, protocol):
     proto = get_object_or_404(MirrorProtocol, protocol=protocol)
     return find_mirrors(request, protocols=[proto])
 
+
+def mail_mirror_admins(data):
+    template = loader.get_template('mirrors/new_mirror_mail_template.txt')
+    for mirror_maintainer in ['[email protected]', '[email protected]']:
+        send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'),
+                      template.render(data),
+                      'Arch Mirror Notification <[email protected]>',
+                      [mirror_maintainer],
+                      fail_silently=True)
+
+
+def submit_mirror(request):
+    if request.method == 'POST' or len(request.GET) > 0:
+        data = request.POST if request.method == 'POST' else request.GET
+
+        # data is immutable, need to be copied and modified to enforce
+        # active and public is False.
+        tmp = data.copy()
+        tmp['active'] = False
+        tmp['public'] = False
+        data = tmp
+
+        mirror_form = MirrorRequestForm(data=data)
+        mirror_url1_form = MirrorUrlForm(data=data, prefix="url1")
+        if data.get('url2-url') != '':
+            mirror_url2_form = MirrorUrlForm(data=data, prefix="url2")
+        else:
+            mirror_url2_form = MirrorUrlForm(prefix="url2")
+        if data.get('url3-url') != '':
+            mirror_url3_form = MirrorUrlForm(data=data, prefix="url3")
+        else:
+            mirror_url3_form = MirrorUrlForm(prefix="url3")
+        rsync_form = MirrorRsyncForm(data=data)
+
+        mirror_url2_form.fields['url'].required = False
+        mirror_url3_form.fields['url'].required = False
+        rsync_form.fields['ip'].required = False
+
+        if mirror_form.is_valid() and mirror_url1_form.is_valid():
+            try:
+                with transaction.atomic():
+                    transaction.on_commit(partial(mail_mirror_admins, data))
+
+                    mirror = mirror_form.save()
+                    mirror_url1 = mirror_url1_form.save(commit=False)
+                    mirror_url1.mirror = mirror
+                    mirror_url1.save()
+
+                    if data.get('url2-url') != '' and mirror_url2_form.is_valid():
+                        mirror_url2 = mirror_url2_form.save(commit=False)
+                        mirror_url2.mirror = mirror
+                        mirror_url2.save()
+                    if data.get('url3-url') != '' and mirror_url3_form.is_valid():
+                        mirror_url3 = mirror_url3_form.save(commit=False)
+                        mirror_url3.mirror = mirror
+                        mirror_url3.save()
+
+                    if data.get('ip') != '' and rsync.is_valid():
+                        rsync = rsync_form.save(commit=False)
+                        rsync.mirror = mirror
+                        rsync.save()
+
+            except DatabaseError as error:
+                print(error)
+
+    else:
+        mirror_form = MirrorRequestForm()
+        mirror_url1_form = MirrorUrlForm(prefix="url1")
+        mirror_url2_form = MirrorUrlForm(prefix="url2")
+        mirror_url3_form = MirrorUrlForm(prefix="url3")
+        rsync_form = MirrorRsyncForm()
+
+        mirror_url2_form.fields['url'].required = False
+        mirror_url3_form.fields['url'].required = False
+        rsync_form.fields['ip'].required = False
+
+    return render(
+        request,
+        'mirrors/mirror_submit.html',
+        {
+            'mirror_form': mirror_form,
+            'mirror_url1_form': mirror_url1_form,
+            'mirror_url2_form': mirror_url2_form,
+            'mirror_url3_form': mirror_url3_form,
+            'rsync_form': rsync_form
+        }
+    )
+
 # vim: set ts=4 sw=4 et:

requirements.txt

@@ -1,4 +1,4 @@
--e git+https://github.com/fredj/cssmin.git@master#egg=cssmin
+cssmin==0.2.0
 Django==4.1.6
 IPy==1.1
 Markdown==3.3.7

templates/mirrors/mirror_submit.html

@@ -0,0 +1,42 @@
+{% extends "base.html" %}
+{% load package_extras %}
+{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %}
+
+{% block content %}
+<div id="mirrorlist-gen" class="box">
+
+    <h2>Mirror Request</h2>
+
+    <p>This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through <a href="https://bugs.archlinux.org/index.php?string=&project=1&search_name=&type%5B%5D=&sev%5B%5D=&pri%5B%5D=&due%5B%5D=&reported%5B%5D=&cat%5B%5D=43&status%5B%5D=open&percent%5B%5D=&opened=&dev=&closed=&duedatefrom=&duedateto=&changedfrom=&changedto=&openedfrom=&openedto=&closedfrom=&closedto=&do=index">https://bugs.archlinux.org</a> and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.</p>
+
+    <h3>Available mirrors</h3>
+
+    <p>Below are direct links to the two different tiers you will need to be acquainted with.<p>
+
+    <ul>
+        <li><a href="/mirrors/tier/1/">Tier 1 mirrors</a></li>
+        <li><a href="/mirrors/tier/2/">Tier 2 mirrors</a></li>
+    </ul>
+
+    <h3>Mirror information</h3>
+
+    {% if mirror_form.is_valid or mirror_url1_form.is_valid %}
+    <code>
+    Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist.
+    </code>
+    {% else %}
+    <p>Before you can submit a <b>Tier 1</b> request the mirror in question must first be a registered <b>Tier 2</b> for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.</p>
+
+    <form id="list-generator" method="get">
+        {{ mirror_form.as_div }}
+        {{ mirror_url1_form.as_div }}
+        {{ mirror_url2_form.as_div }}
+        {{ mirror_url3_form.as_div }}
+        <p>If you are registring a <b>Tier 1</b> mirror, you need to supply the static IP to the machine that will be using rsync towards the Tier 0 mirror. This is so we can allow it to sync.
+        {{ rsync_form.as_div }}
+        <p><label></label> <input type="submit" value="Submit Request" /></p>
+    </form>
+    {% endif %}
+</div>
+{% endblock %}
+

templates/mirrors/new_mirror_mail_template.txt

@@ -0,0 +1,9 @@
+{% autoescape off %}A new mirror has been requested called "{{ name }}". As the mirror administrator, check in on https://archlinux.org/mirrors/{{ name }}/ after a few minutes and check:
+
+ * Is the Completion % more than 98% for Tier 2 and 100% for Tier 1?
+ * Does it have HTTP or HTTPS protocols?
+
+If so, go to https://archlinux.org/admin/mirrors/, find the new mirror and mark it as Active and Public!
+If the mirror hasn't synced in a couple of days, email the mirror admin asking if we can assist in any way.
+
+{% endautoescape %}