feat: provision to use local mirror

press/docker/Dockerfile

@@ -7,6 +7,12 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV OPENBLAS_NUM_THREADS 1
 ENV MKL_NUM_THREADS 1
 
+#replace archive.ubuntu.com by mirror server in source.list
+{% if doc.mirror_server %}
+RUN sed -i -e "s/archive.ubuntu.com/{{doc.mirror_server}}/" /etc/apt/sources.list
+RUN sed -i -e "s/security.ubuntu.com/{{doc.mirror_server}}/" /etc/apt/sources.list
+{% endif %}
+
 # Install essential packages
 RUN --mount=type=cache,target=/var/cache/apt apt-get update \
  && apt-get install --yes --no-install-suggests --no-install-recommends \

press/playbooks/mirror.yml

@@ -0,0 +1,22 @@
+---
+- name: Setup Database Server
+ hosts: all
+ become: yes
+ become_user: root
+ gather_facts: yes
+ roles:
+ - role: essentials
+ - role: user
+ - role: nginx
+ - role: agent
+ - role: mirror
+ - role: node_exporter
+ - role: deadlock_logger
+ - role: filebeat
+ - role: clamav
+ - role: aide
+ - role: additional_process_hardening
+ - role: warning_banners
+ - role: auditd
+ - role: sshd_hardening
+ - role: pam

press/playbooks/roles/mirror/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- name: Create Mirror Directories
+ become: yes
+ become_user: frappe
+ file:
+ path: /home/frappe/ubuntu/
+ state: directory
+ mode: 0755
+
+- name: Install apt-mirror
+ apt:
+ package: apt-mirror
+ state: present
+
+- name: Test for apt-mirror access via frappe user
+ shell: grep -c "apt-mirror" /etc/sudoers.d/frappe || true
+ register: apt_mirror_permissions
+
+- name: Update sudoers
+ lineinfile:
+ dest: /etc/sudoers.d/frappe
+ line: 'frappe ALL = (root) NOPASSWD: /usr/bin/apt-mirror'
+ when: apt_mirror_permissions.stdout == '0'
+
+- name: Backup Mirror List
+ copy:
+ src: /etc/apt/mirror.list
+ dest: /etc/apt/mirror.list-back
+ remote_src: true
+
+- name: Setup mirror config
+ template:
+ src: mirror.list.j2
+ dest: /etc/apt/mirror.list
+
+- name: Start mirroring
+ become: yes
+ become_user: frappe
+ shell: 'nohup apt-mirror > nohup.out 2>&1 &'
+ args:
+ chdir: /home/frappe
+ executable: /bin/bash

press/playbooks/roles/mirror/templates/mirror.list.j2

@@ -0,0 +1,29 @@
+############# config ##################
+#
+set base_path /home/frappe/ubuntu
+#
+set mirror_path $base_path/mirror
+set skel_path $base_path/skel
+set var_path $base_path/var
+set cleanscript $var_path/clean.sh
+# set defaultarch <running host architecture>
+# set postmirror_script $var_path/postmirror.sh
+# set run_postmirror 0
+set nthreads 20
+set _tilde 0
+#
+############# end config ##############
+
+deb http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse
+deb http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse
+deb http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse
+deb http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse
+deb http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse
+
+# deb-src http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse
+# deb-src http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse
+# deb-src http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse
+# deb-src http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse
+# deb-src http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse
+
+clean http://archive.ubuntu.com/ubuntu

press/press/doctype/deploy_candidate/deploy_candidate.json

@@ -7,6 +7,7 @@
  "field_order": [
  "status",
  "is_remote_builder_used",
+ "mirror_server",
  "column_break_2",
  "group",
  "team",
@@ -406,6 +407,12 @@
  "fieldtype": "Int",
  "label": "Retry Count",
  "read_only": 1
+ },
+ {
+ "fieldname": "mirror_server",
+ "fieldtype": "Link",
+ "label": "Mirror Server",
+ "options": "Mirror Server"
  }
  ],
  "links": [
@@ -422,7 +429,7 @@
  "link_fieldname": "document_name"
  }
  ],
- "modified": "2024-05-24 11:50:55.127138",
+ "modified": "2024-06-05 11:13:35.702377",
  "modified_by": "Administrator",
  "module": "Press",
  "name": "Deploy Candidate",

press/press/doctype/deploy_candidate/deploy_candidate.py

@@ -108,6 +108,7 @@ class DeployCandidate(Document):
  manually_failed: DF.Check
  merge_all_rq_queues: DF.Check
  merge_default_and_short_rq_queues: DF.Check
+ mirror_server: DF.Link | None
  packages: DF.Table[DeployCandidatePackage]
  pending_duration: DF.Time | None
  pending_end: DF.Datetime | None

press/press/doctype/mirror_server/mirror_server.js

@@ -0,0 +1,51 @@
+// Copyright (c) 2024, Frappe and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Mirror Server', {
+ refresh(frm) {
+ frm.add_web_link(
+ `/dashboard/servers/${frm.doc.name}`,
+ __('Visit Dashboard'),
+ );
+
+ [
+ [
+ __('Prepare Server'),
+ 'prepare_server',
+ true,
+ !frm.doc.is_server_prepared,
+ ],
+ [__('Setup Server'), 'setup_server', true, !frm.doc.is_server_setup],
+ ].forEach(([label, method, confirm, condition]) => {
+ if (typeof condition === 'undefined' || condition) {
+ frm.add_custom_button(
+ label,
+ () => {
+ if (confirm) {
+ frappe.confirm(
+ `Are you sure you want to ${label.toLowerCase()}?`,
+ () =>
+ frm.call(method).then((r) => {
+ if (r.message) {
+ frappe.msgprint(r.message);
+ } else {
+ frm.refresh();
+ }
+ }),
+ );
+ } else {
+ frm.call(method).then((r) => {
+ if (r.message) {
+ frappe.msgprint(r.message);
+ } else {
+ frm.refresh();
+ }
+ });
+ }
+ },
+ __('Actions'),
+ );
+ }
+ });
+ },
+});

press/press/doctype/mirror_server/mirror_server.json

@@ -0,0 +1,194 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-25 17:09:27.662072",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+ "status",
+ "hostname",
+ "domain",
+ "column_break_ujmi",
+ "cluster",
+ "provider",
+ "virtual_machine",
+ "is_server_setup",
+ "networking_section",
+ "ip",
+ "column_break_mrkw",
+ "private_ip",
+ "mirror_info_section",
+ "os",
+ "mirror_sync_status",
+ "column_break_zjsf",
+ "os_version",
+ "agent_section",
+ "agent_password",
+ "ssh_section",
+ "ssh_user",
+ "ssh_port",
+ "frappe_user_password",
+ "frappe_public_key",
+ "column_break_cgrf",
+ "root_public_key"
+ ],
+ "fields": [
+ {
+ "fieldname": "status",
+ "fieldtype": "Select",
+ "label": "Status",
+ "options": "Pending\nInstalling\nActive\nBroken\nArchived"
+ },
+ {
+ "fieldname": "hostname",
+ "fieldtype": "Data",
+ "label": "Hostname"
+ },
+ {
+ "fieldname": "domain",
+ "fieldtype": "Link",
+ "label": "Domain",
+ "options": "Domain"
+ },
+ {
+ "fieldname": "column_break_ujmi",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "cluster",
+ "fieldtype": "Link",
+ "label": "Cluster",
+ "options": "Cluster"
+ },
+ {
+ "fieldname": "provider",
+ "fieldtype": "Select",
+ "label": "Provider",
+ "options": "Generic\nScaleway\nAWS EC2\nOCI"
+ },
+ {
+ "fieldname": "virtual_machine",
+ "fieldtype": "Link",
+ "label": "Virtual Machine",
+ "options": "Virtual Machine"
+ },
+ {
+ "default": "0",
+ "fieldname": "is_server_setup",
+ "fieldtype": "Check",
+ "label": "Server Setup"
+ },
+ {
+ "fieldname": "networking_section",
+ "fieldtype": "Section Break",
+ "label": "Networking"
+ },
+ {
+ "fieldname": "column_break_mrkw",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "private_ip",
+ "fieldtype": "Data",
+ "label": "Private IP"
+ },
+ {
+ "fieldname": "agent_section",
+ "fieldtype": "Section Break",
+ "label": "Agent"
+ },
+ {
+ "fieldname": "agent_password",
+ "fieldtype": "Password",
+ "label": "Agent Password"
+ },
+ {
+ "fieldname": "ssh_section",
+ "fieldtype": "Section Break",
+ "label": "SSH"
+ },
+ {
+ "fieldname": "ssh_user",
+ "fieldtype": "Data",
+ "label": "SSH User"
+ },
+ {
+ "fieldname": "ssh_port",
+ "fieldtype": "Int",
+ "label": "SSH Port"
+ },
+ {
+ "fieldname": "frappe_user_password",
+ "fieldtype": "Data",
+ "label": "Frappe User Password"
+ },
+ {
+ "fieldname": "frappe_public_key",
+ "fieldtype": "Code",
+ "label": "Frappe Public Key"
+ },
+ {
+ "fieldname": "column_break_cgrf",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "root_public_key",
+ "fieldtype": "Code",
+ "label": "Root Public Key"
+ },
+ {
+ "fieldname": "os",
+ "fieldtype": "Data",
+ "label": "OS"
+ },
+ {
+ "fieldname": "column_break_zjsf",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "os_version",
+ "fieldtype": "Data",
+ "label": "OS Version"
+ },
+ {
+ "fieldname": "ip",
+ "fieldtype": "Data",
+ "label": "Public IP"
+ },
+ {
+ "fieldname": "mirror_info_section",
+ "fieldtype": "Section Break",
+ "label": "Mirror Info"
+ },
+ {
+ "fieldname": "mirror_sync_status",
+ "fieldtype": "Select",
+ "label": "Mirror Sync Status",
+ "options": "Syncing\nCompleted\nBroken"
+ }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2024-05-06 12:23:43.249066",
+ "modified_by": "Administrator",
+ "module": "Press",
+ "name": "Mirror Server",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "share": 1,
+ "write": 1
+ }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}