Automate mirror submission

.gitignore

@@ -6,6 +6,7 @@ local_settings.py
 archweb.db
 archweb.db-*
 database.db
+/*.tar.gz
 tags
 collected_static/
 testing/

mirrors/urls_mirrorlist.py

@@ -3,6 +3,7 @@
 
 urlpatterns = [
  path('', views.generate_mirrorlist, name='mirrorlist'),
+ path('submit/', views.submit_mirror, name='mirrorsubmit'),
  re_path(r'^all/$', views.find_mirrors, {'countries': ['all']}),
  re_path(r'^all/(?P<protocol>[A-z]+)/$', views.find_mirrors_simple, name='mirrorlist_simple')
 ]

mirrors/views/mirrorlist.py

@@ -1,17 +1,132 @@
 from operator import attrgetter, itemgetter
+from urllib.parse import urlparse, urlunsplit
+from functools import partial
 
 from django import forms
+from django.db import DatabaseError, transaction
 from django.db.models import Q
-from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
+from django.core.mail import send_mail
+from django.template import loader
+from django.forms.widgets import (
+ Select,
+ SelectMultiple,
+ CheckboxSelectMultiple,
+ TextInput,
+ EmailInput
+)
 from django.shortcuts import get_object_or_404, redirect, render
 from django.views.decorators.csrf import csrf_exempt
 from django_countries import countries
 
-from ..models import MirrorUrl, MirrorProtocol
+from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync
 from ..utils import get_mirror_statuses
 
 import random
 
+# This is populated later, and re-populated every refresh
+# This was the only way to get 3 different examples without
+# changing the models.py
+url_examples = []
+
+
+class MirrorRequestForm(forms.ModelForm):
+ upstream = forms.ModelChoiceField(
+ queryset=Mirror.objects.filter(
+ tier__gte=0,
+ tier__lte=1
+ ),
+ required=False
+ )
+
+ class Meta:
+ model = Mirror
+ fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email',
+ 'isos', 'active', 'public', 'rsync_user', 'rsync_password', 'notes')
+
+ def __init__(self, *args, **kwargs):
+ super(MirrorRequestForm, self).__init__(*args, **kwargs)
+ fields = self.fields
+ fields['name'].widget.attrs.update({'placeholder': 'Ex: mirror.argentina.co'})
+ fields['rsync_user'].widget.attrs.update({'placeholder': 'Optional'})
+ fields['rsync_password'].widget.attrs.update({'placeholder': 'Optional'})
+ fields['notes'].widget.attrs.update({'placeholder': 'Ex: Hosted by ISP GreatISO.bg'})
+
+ def as_div(self):
+ "Returns this form rendered as HTML <divs>s."
+ return self._html_output(
+ normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+ error_row=u'%s',
+ row_ender='</div>',
+ help_text_html=u' <span class="helptext">%s</span>',
+ errors_on_separate_row=True)
+
+
+class MirrorUrlForm(forms.ModelForm):
+ class Meta:
+ model = MirrorUrl
+ fields = ('url', 'country', 'bandwidth', 'active')
+
+ def __init__(self, *args, **kwargs):
+ global url_examples
+
+ super(MirrorUrlForm, self).__init__(*args, **kwargs)
+ fields = self.fields
+
+ if len(url_examples) == 0:
+ url_examples = [
+ 'Ex: http://mirror.argentina.co/archlinux',
+ 'Ex: https://mirror.argentina.co/archlinux',
+ 'Ex: rsync://mirror.argentina.co/archlinux'
+ ]
+
+ fields['url'].widget.attrs.update({'placeholder': url_examples.pop()})
+
+ def clean_url(self):
+ # is this a valid-looking URL?
+ url_parts = urlparse(self.cleaned_data["url"])
+ if not url_parts.scheme:
+ raise forms.ValidationError("No URL scheme (protocol) provided.")
+ if not url_parts.netloc:
+ raise forms.ValidationError("No URL host provided.")
+ if url_parts.params or url_parts.query or url_parts.fragment:
+ raise forms.ValidationError(
+ "URL parameters, query, and fragment elements are not supported.")
+ # ensure we always save the URL with a trailing slash
+ path = url_parts.path
+ if not path.endswith('/'):
+ path += '/'
+ url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', ''))
+ return url
+
+ def as_div(self):
+ "Returns this form rendered as HTML <divs>s."
+ return self._html_output(
+ normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+ error_row=u'%s',
+ row_ender='</div>',
+ help_text_html=u' <span class="helptext">%s</span>',
+ errors_on_separate_row=True)
+
+
+class MirrorRsyncForm(forms.ModelForm):
+ class Meta:
+ model = MirrorRsync
+ fields = ('ip',)
+
+ def __init__(self, *args, **kwargs):
+ super(MirrorRsyncForm, self).__init__(*args, **kwargs)
+ fields = self.fields
+ fields['ip'].widget.attrs.update({'placeholder': 'Ex: 1.2.4.5'})
+
+ def as_div(self):
+ "Returns this form rendered as HTML <divs>s."
+ return self._html_output(
+ normal_row=u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+ error_row=u'%s',
+ row_ender='</div>',
+ help_text_html=u' <span class="helptext">%s</span>',
+ errors_on_separate_row=True)
+
 
 class MirrorlistForm(forms.Form):
  country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'}))
@@ -127,4 +242,92 @@ def find_mirrors_simple(request, protocol):
  proto = get_object_or_404(MirrorProtocol, protocol=protocol)
  return find_mirrors(request, protocols=[proto])
 
+
+def mail_mirror_admins(data):
+ template = loader.get_template('mirrors/new_mirror_mail_template.txt')
+ for mirror_maintainer in ['[email protected]', '[email protected]']:
+ send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'),
+ template.render(data),
+ 'Arch Mirror Notification <[email protected]>',
+ [mirror_maintainer],
+ fail_silently=True)
+
+
+def submit_mirror(request):
+ if request.method == 'POST' or len(request.GET) > 0:
+ data = request.POST if request.method == 'POST' else request.GET
+
+ # data is immutable, need to be copied and modified to enforce
+ # active and public is False.
+ tmp = data.copy()
+ tmp['active'] = False
+ tmp['public'] = False
+ data = tmp
+
+ mirror_form = MirrorRequestForm(data=data)
+ mirror_url1_form = MirrorUrlForm(data=data, prefix="url1")
+ if data.get('url2-url') != '':
+ mirror_url2_form = MirrorUrlForm(data=data, prefix="url2")
+ else:
+ mirror_url2_form = MirrorUrlForm(prefix="url2")
+ if data.get('url3-url') != '':
+ mirror_url3_form = MirrorUrlForm(data=data, prefix="url3")
+ else:
+ mirror_url3_form = MirrorUrlForm(prefix="url3")
+ rsync_form = MirrorRsyncForm(data=data)
+
+ mirror_url2_form.fields['url'].required = False
+ mirror_url3_form.fields['url'].required = False
+ rsync_form.fields['ip'].required = False
+
+ if mirror_form.is_valid() and mirror_url1_form.is_valid():
+ try:
+ with transaction.atomic():
+ transaction.on_commit(partial(mail_mirror_admins, data))
+
+ mirror = mirror_form.save()
+ mirror_url1 = mirror_url1_form.save(commit=False)
+ mirror_url1.mirror = mirror
+ mirror_url1.save()
+
+ if data.get('url2-url') != '' and mirror_url2_form.is_valid():
+ mirror_url2 = mirror_url2_form.save(commit=False)
+ mirror_url2.mirror = mirror
+ mirror_url2.save()
+ if data.get('url3-url') != '' and mirror_url3_form.is_valid():
+ mirror_url3 = mirror_url3_form.save(commit=False)
+ mirror_url3.mirror = mirror
+ mirror_url3.save()
+
+ if data.get('ip') != '' and rsync.is_valid():
+ rsync = rsync_form.save(commit=False)
+ rsync.mirror = mirror
+ rsync.save()
+
+ except DatabaseError as error:
+ print(error)
+
+ else:
+ mirror_form = MirrorRequestForm()
+ mirror_url1_form = MirrorUrlForm(prefix="url1")
+ mirror_url2_form = MirrorUrlForm(prefix="url2")
+ mirror_url3_form = MirrorUrlForm(prefix="url3")
+ rsync_form = MirrorRsyncForm()
+
+ mirror_url2_form.fields['url'].required = False
+ mirror_url3_form.fields['url'].required = False
+ rsync_form.fields['ip'].required = False
+
+ return render(
+ request,
+ 'mirrors/mirror_submit.html',
+ {
+ 'mirror_form': mirror_form,
+ 'mirror_url1_form': mirror_url1_form,
+ 'mirror_url2_form': mirror_url2_form,
+ 'mirror_url3_form': mirror_url3_form,
+ 'rsync_form': rsync_form
+ }
+ )
+
 # vim: set ts=4 sw=4 et:

requirements.txt

@@ -1,4 +1,4 @@
--e git+https://github.com/fredj/cssmin.git@master#egg=cssmin
+cssmin==0.2.0
 Django==4.1.6
 IPy==1.1
 Markdown==3.3.7

templates/mirrors/mirror_submit.html

@@ -0,0 +1,42 @@
+{% extends "base.html" %}
+{% load package_extras %}
+{% block title %}Arch Linux - Pacman Mirrorlist Generator{% endblock %}
+
+{% block content %}
+<div id="mirrorlist-gen" class="box">
+
+ <h2>Mirror Request</h2>
+
+ <p>This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through <a href="https://bugs.archlinux.org/index.php?string=&project=1&search_name=&type%5B%5D=&sev%5B%5D=&pri%5B%5D=&due%5B%5D=&reported%5B%5D=&cat%5B%5D=43&status%5B%5D=open&percent%5B%5D=&opened=&dev=&closed=&duedatefrom=&duedateto=&changedfrom=&changedto=&openedfrom=&openedto=&closedfrom=&closedto=&do=index">https://bugs.archlinux.org</a> and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.</p>
+
+ <h3>Available mirrors</h3>
+
+ <p>Below are direct links to the two different tiers you will need to be acquainted with.<p>
+
+ <ul>
+ <li><a href="/mirrors/tier/1/">Tier 1 mirrors</a></li>
+ <li><a href="/mirrors/tier/2/">Tier 2 mirrors</a></li>
+ </ul>
+
+ <h3>Mirror information</h3>
+
+ {% if mirror_form.is_valid or mirror_url1_form.is_valid %}
+ <code>
+ Your request have successfully been submitted and should be visible within 5 minutes in the mirrorlist.
+ </code>
+ {% else %}
+ <p>Before you can submit a <b>Tier 1</b> request the mirror in question must first be a registered <b>Tier 2</b> for a certain amount of time with proven reliablity. Once the submitted information is verified the mirror will be visible under the appropriate tier list above. This process usually takes 5 minutes.</p>
+
+ <form id="list-generator" method="get">
+ {{ mirror_form.as_div }}
+ {{ mirror_url1_form.as_div }}
+ {{ mirror_url2_form.as_div }}
+ {{ mirror_url3_form.as_div }}
+ <p>If you are registring a <b>Tier 1</b> mirror, you need to supply the static IP to the machine that will be using rsync towards the Tier 0 mirror. This is so we can allow it to sync.
+ {{ rsync_form.as_div }}
+ <p><label></label> <input type="submit" value="Submit Request" /></p>
+ </form>
+ {% endif %}
+</div>
+{% endblock %}
+

templates/mirrors/new_mirror_mail_template.txt

@@ -0,0 +1,9 @@
+{% autoescape off %}A new mirror has been requested called "{{ name }}". As the mirror administrator, check in on https://archlinux.org/mirrors/{{ name }}/ after a few minutes and check:
+
+ * Is the Completion % more than 98% for Tier 2 and 100% for Tier 1?
+ * Does it have HTTP or HTTPS protocols?
+
+If so, go to https://archlinux.org/admin/mirrors/, find the new mirror and mark it as Active and Public!
+If the mirror hasn't synced in a couple of days, email the mirror admin asking if we can assist in any way.
+
+{% endautoescape %}