diff --git a/press/docker/Dockerfile b/press/docker/Dockerfile index 46981b933a..8fafc99e5f 100644 --- a/press/docker/Dockerfile +++ b/press/docker/Dockerfile @@ -7,6 +7,12 @@ ENV DEBIAN_FRONTEND noninteractive ENV OPENBLAS_NUM_THREADS 1 ENV MKL_NUM_THREADS 1 +#replace archive.ubuntu.com by mirror server in source.list +{% if doc.mirror_server %} +RUN sed -i -e "s/archive.ubuntu.com/{{doc.mirror_server}}/" /etc/apt/sources.list +RUN sed -i -e "s/security.ubuntu.com/{{doc.mirror_server}}/" /etc/apt/sources.list +{% endif %} + # Install essential packages RUN --mount=type=cache,target=/var/cache/apt apt-get update \ && apt-get install --yes --no-install-suggests --no-install-recommends \ diff --git a/press/playbooks/mirror.yml b/press/playbooks/mirror.yml new file mode 100644 index 0000000000..b11307560f --- /dev/null +++ b/press/playbooks/mirror.yml @@ -0,0 +1,22 @@ +--- +- name: Setup Database Server + hosts: all + become: yes + become_user: root + gather_facts: yes + roles: + - role: essentials + - role: user + - role: nginx + - role: agent + - role: mirror + - role: node_exporter + - role: deadlock_logger + - role: filebeat + - role: clamav + - role: aide + - role: additional_process_hardening + - role: warning_banners + - role: auditd + - role: sshd_hardening + - role: pam diff --git a/press/playbooks/roles/mirror/tasks/main.yml b/press/playbooks/roles/mirror/tasks/main.yml new file mode 100644 index 0000000000..754ff28d3b --- /dev/null +++ b/press/playbooks/roles/mirror/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Create Mirror Directories + become: yes + become_user: frappe + file: + path: /home/frappe/ubuntu/ + state: directory + mode: 0755 + +- name: Install apt-mirror + apt: + package: apt-mirror + state: present + +- name: Test for apt-mirror access via frappe user + shell: grep -c "apt-mirror" /etc/sudoers.d/frappe || true + register: apt_mirror_permissions + +- name: Update sudoers + lineinfile: + dest: /etc/sudoers.d/frappe + line: 'frappe ALL = (root) NOPASSWD: /usr/bin/apt-mirror' + when: apt_mirror_permissions.stdout == '0' + +- name: Backup Mirror List + copy: + src: /etc/apt/mirror.list + dest: /etc/apt/mirror.list-back + remote_src: true + +- name: Setup mirror config + template: + src: mirror.list.j2 + dest: /etc/apt/mirror.list + +- name: Start mirroring + become: yes + become_user: frappe + shell: 'nohup apt-mirror > nohup.out 2>&1 &' + args: + chdir: /home/frappe + executable: /bin/bash \ No newline at end of file diff --git a/press/playbooks/roles/mirror/templates/mirror.list.j2 b/press/playbooks/roles/mirror/templates/mirror.list.j2 new file mode 100644 index 0000000000..c5933b119b --- /dev/null +++ b/press/playbooks/roles/mirror/templates/mirror.list.j2 @@ -0,0 +1,29 @@ +############# config ################## +# +set base_path /home/frappe/ubuntu +# +set mirror_path $base_path/mirror +set skel_path $base_path/skel +set var_path $base_path/var +set cleanscript $var_path/clean.sh +# set defaultarch +# set postmirror_script $var_path/postmirror.sh +# set run_postmirror 0 +set nthreads 20 +set _tilde 0 +# +############# end config ############## + +deb http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse + +# deb-src http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse + +clean http://archive.ubuntu.com/ubuntu \ No newline at end of file diff --git a/press/press/doctype/deploy_candidate/deploy_candidate.json b/press/press/doctype/deploy_candidate/deploy_candidate.json index 1549e98a62..db509f2c42 100644 --- a/press/press/doctype/deploy_candidate/deploy_candidate.json +++ b/press/press/doctype/deploy_candidate/deploy_candidate.json @@ -7,6 +7,7 @@ "field_order": [ "status", "is_remote_builder_used", + "mirror_server", "column_break_2", "group", "team", @@ -406,6 +407,12 @@ "fieldtype": "Int", "label": "Retry Count", "read_only": 1 + }, + { + "fieldname": "mirror_server", + "fieldtype": "Link", + "label": "Mirror Server", + "options": "Mirror Server" } ], "links": [ @@ -422,7 +429,7 @@ "link_fieldname": "document_name" } ], - "modified": "2024-05-24 11:50:55.127138", + "modified": "2024-06-05 11:13:35.702377", "modified_by": "Administrator", "module": "Press", "name": "Deploy Candidate", diff --git a/press/press/doctype/deploy_candidate/deploy_candidate.py b/press/press/doctype/deploy_candidate/deploy_candidate.py index c1eef960e3..b4022e6f54 100644 --- a/press/press/doctype/deploy_candidate/deploy_candidate.py +++ b/press/press/doctype/deploy_candidate/deploy_candidate.py @@ -108,6 +108,7 @@ class DeployCandidate(Document): manually_failed: DF.Check merge_all_rq_queues: DF.Check merge_default_and_short_rq_queues: DF.Check + mirror_server: DF.Link | None packages: DF.Table[DeployCandidatePackage] pending_duration: DF.Time | None pending_end: DF.Datetime | None diff --git a/press/press/doctype/mirror_server/__init__.py b/press/press/doctype/mirror_server/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/press/press/doctype/mirror_server/mirror_server.js b/press/press/doctype/mirror_server/mirror_server.js new file mode 100644 index 0000000000..ca532204d0 --- /dev/null +++ b/press/press/doctype/mirror_server/mirror_server.js @@ -0,0 +1,51 @@ +// Copyright (c) 2024, Frappe and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Mirror Server', { + refresh(frm) { + frm.add_web_link( + `/dashboard/servers/${frm.doc.name}`, + __('Visit Dashboard'), + ); + + [ + [ + __('Prepare Server'), + 'prepare_server', + true, + !frm.doc.is_server_prepared, + ], + [__('Setup Server'), 'setup_server', true, !frm.doc.is_server_setup], + ].forEach(([label, method, confirm, condition]) => { + if (typeof condition === 'undefined' || condition) { + frm.add_custom_button( + label, + () => { + if (confirm) { + frappe.confirm( + `Are you sure you want to ${label.toLowerCase()}?`, + () => + frm.call(method).then((r) => { + if (r.message) { + frappe.msgprint(r.message); + } else { + frm.refresh(); + } + }), + ); + } else { + frm.call(method).then((r) => { + if (r.message) { + frappe.msgprint(r.message); + } else { + frm.refresh(); + } + }); + } + }, + __('Actions'), + ); + } + }); + }, +}); diff --git a/press/press/doctype/mirror_server/mirror_server.json b/press/press/doctype/mirror_server/mirror_server.json new file mode 100644 index 0000000000..94fd9fdbef --- /dev/null +++ b/press/press/doctype/mirror_server/mirror_server.json @@ -0,0 +1,194 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-04-25 17:09:27.662072", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "status", + "hostname", + "domain", + "column_break_ujmi", + "cluster", + "provider", + "virtual_machine", + "is_server_setup", + "networking_section", + "ip", + "column_break_mrkw", + "private_ip", + "mirror_info_section", + "os", + "mirror_sync_status", + "column_break_zjsf", + "os_version", + "agent_section", + "agent_password", + "ssh_section", + "ssh_user", + "ssh_port", + "frappe_user_password", + "frappe_public_key", + "column_break_cgrf", + "root_public_key" + ], + "fields": [ + { + "fieldname": "status", + "fieldtype": "Select", + "label": "Status", + "options": "Pending\nInstalling\nActive\nBroken\nArchived" + }, + { + "fieldname": "hostname", + "fieldtype": "Data", + "label": "Hostname" + }, + { + "fieldname": "domain", + "fieldtype": "Link", + "label": "Domain", + "options": "Domain" + }, + { + "fieldname": "column_break_ujmi", + "fieldtype": "Column Break" + }, + { + "fieldname": "cluster", + "fieldtype": "Link", + "label": "Cluster", + "options": "Cluster" + }, + { + "fieldname": "provider", + "fieldtype": "Select", + "label": "Provider", + "options": "Generic\nScaleway\nAWS EC2\nOCI" + }, + { + "fieldname": "virtual_machine", + "fieldtype": "Link", + "label": "Virtual Machine", + "options": "Virtual Machine" + }, + { + "default": "0", + "fieldname": "is_server_setup", + "fieldtype": "Check", + "label": "Server Setup" + }, + { + "fieldname": "networking_section", + "fieldtype": "Section Break", + "label": "Networking" + }, + { + "fieldname": "column_break_mrkw", + "fieldtype": "Column Break" + }, + { + "fieldname": "private_ip", + "fieldtype": "Data", + "label": "Private IP" + }, + { + "fieldname": "agent_section", + "fieldtype": "Section Break", + "label": "Agent" + }, + { + "fieldname": "agent_password", + "fieldtype": "Password", + "label": "Agent Password" + }, + { + "fieldname": "ssh_section", + "fieldtype": "Section Break", + "label": "SSH" + }, + { + "fieldname": "ssh_user", + "fieldtype": "Data", + "label": "SSH User" + }, + { + "fieldname": "ssh_port", + "fieldtype": "Int", + "label": "SSH Port" + }, + { + "fieldname": "frappe_user_password", + "fieldtype": "Data", + "label": "Frappe User Password" + }, + { + "fieldname": "frappe_public_key", + "fieldtype": "Code", + "label": "Frappe Public Key" + }, + { + "fieldname": "column_break_cgrf", + "fieldtype": "Column Break" + }, + { + "fieldname": "root_public_key", + "fieldtype": "Code", + "label": "Root Public Key" + }, + { + "fieldname": "os", + "fieldtype": "Data", + "label": "OS" + }, + { + "fieldname": "column_break_zjsf", + "fieldtype": "Column Break" + }, + { + "fieldname": "os_version", + "fieldtype": "Data", + "label": "OS Version" + }, + { + "fieldname": "ip", + "fieldtype": "Data", + "label": "Public IP" + }, + { + "fieldname": "mirror_info_section", + "fieldtype": "Section Break", + "label": "Mirror Info" + }, + { + "fieldname": "mirror_sync_status", + "fieldtype": "Select", + "label": "Mirror Sync Status", + "options": "Syncing\nCompleted\nBroken" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-05-06 12:23:43.249066", + "modified_by": "Administrator", + "module": "Press", + "name": "Mirror Server", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/press/press/doctype/mirror_server/mirror_server.py b/press/press/doctype/mirror_server/mirror_server.py new file mode 100644 index 0000000000..56670f6ea9 --- /dev/null +++ b/press/press/doctype/mirror_server/mirror_server.py @@ -0,0 +1,95 @@ +# Copyright (c) 2024, Frappe and contributors +# For license information, please see license.txt + +import frappe + + +from press.press.doctype.server.server import BaseServer +from press.runner import Ansible +from press.utils import log_error + + +class MirrorServer(BaseServer): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + agent_password: DF.Password | None + cluster: DF.Link | None + domain: DF.Link | None + frappe_public_key: DF.Code | None + frappe_user_password: DF.Data | None + hostname: DF.Data | None + ip: DF.Data | None + is_server_setup: DF.Check + mirror_sync_status: DF.Literal["Syncing", "Completed", "Broken"] + os: DF.Data | None + os_version: DF.Data | None + private_ip: DF.Data | None + provider: DF.Literal["Generic", "Scaleway", "AWS EC2", "OCI"] + root_public_key: DF.Code | None + ssh_port: DF.Int + ssh_user: DF.Data | None + status: DF.Literal["Pending", "Installing", "Active", "Broken", "Archived"] + virtual_machine: DF.Link | None + # end: auto-generated types + + def validate(self): + self.validate_agent_password() + + def _setup_server(self): + agent_password = self.get_password("agent_password") + agent_repository_url = self.get_agent_repository_url() + certificate_name = frappe.db.get_value( + "TLS Certificate", {"wildcard": True, "domain": self.domain}, "name" + ) + certificate = frappe.get_doc("TLS Certificate", certificate_name) + monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password( + "monitoring_password" + ) + + log_server = frappe.db.get_single_value("Press Settings", "log_server") + if log_server: + kibana_password = frappe.get_doc("Log Server", log_server).get_password( + "kibana_password" + ) + else: + kibana_password = None + + try: + ansible = Ansible( + playbook="mirror.yml", + server=self, + user=self.ssh_user or "root", + port=self.ssh_port or 22, + variables={ + "server": self.name, + "workers": "1", + "mirror_server": True, + "agent_password": agent_password, + "agent_repository_url": agent_repository_url, + "monitoring_password": monitoring_password, + "log_server": log_server, + "kibana_password": kibana_password, + "private_ip": self.private_ip, + "certificate_private_key": certificate.private_key, + "certificate_full_chain": certificate.full_chain, + "certificate_intermediate_chain": certificate.intermediate_chain, + }, + ) + play = ansible.run() + self.reload() + if play.status == "Success": + self.status = "Active" + self.is_server_setup = True + + else: + self.status = "Broken" + except Exception: + self.status = "Broken" + log_error("Mirror Server Setup Exception", server=self.as_dict()) + self.save() diff --git a/press/press/doctype/mirror_server/mirror_server_dashboard.py b/press/press/doctype/mirror_server/mirror_server_dashboard.py new file mode 100644 index 0000000000..2f12a9126c --- /dev/null +++ b/press/press/doctype/mirror_server/mirror_server_dashboard.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and Contributors +# For license information, please see license.txt + + +from frappe import _ + + +def get_data(): + return { + "fieldname": "server", + "transactions": [ + {"label": _("Logs"), "items": ["Agent Job", "Ansible Play"]}, + ], + } diff --git a/press/press/doctype/mirror_server/test_mirror_server.py b/press/press/doctype/mirror_server/test_mirror_server.py new file mode 100644 index 0000000000..e6b71b137b --- /dev/null +++ b/press/press/doctype/mirror_server/test_mirror_server.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestMirrorServer(FrappeTestCase): + pass diff --git a/press/press/doctype/release_group/release_group.json b/press/press/doctype/release_group/release_group.json index ab923a3ee5..6e318402cb 100644 --- a/press/press/doctype/release_group/release_group.json +++ b/press/press/doctype/release_group/release_group.json @@ -15,6 +15,8 @@ "default", "public", "central_bench", + "use_local_mirror", + "mirror_server", "section_break_7", "servers", "section_break_keov", @@ -367,10 +369,24 @@ "fieldtype": "Link", "label": "Remote Build Server", "options": "Server" + }, + { + "default": "0", + "description": "If enabled, it will download Ubuntu packages from the local mirror", + "fieldname": "use_local_mirror", + "fieldtype": "Check", + "label": "Use Local Mirror" + }, + { + "depends_on": "eval:doc.use_local_mirror", + "fieldname": "mirror_server", + "fieldtype": "Link", + "label": "Mirror Server", + "options": "Mirror Server" } ], "links": [], - "modified": "2024-05-23 13:53:01.038192", + "modified": "2024-04-26 11:31:23.254144", "modified_by": "Administrator", "module": "Press", "name": "Release Group", diff --git a/press/press/doctype/release_group/release_group.py b/press/press/doctype/release_group/release_group.py index e71e7bf9f3..76057b1b40 100644 --- a/press/press/doctype/release_group/release_group.py +++ b/press/press/doctype/release_group/release_group.py @@ -102,6 +102,7 @@ class ReleaseGroup(Document, TagHelpers): merge_default_and_short_rq_queues: DF.Check min_background_workers: DF.Int min_gunicorn_workers: DF.Int + mirror_server: DF.Link | None mounts: DF.Table[ReleaseGroupMount] packages: DF.Table[ReleaseGroupPackage] public: DF.Check @@ -114,6 +115,7 @@ class ReleaseGroup(Document, TagHelpers): title: DF.Data use_app_cache: DF.Check use_delta_builds: DF.Check + use_local_mirror: DF.Check use_rq_workerpool: DF.Check version: DF.Link # end: auto-generated types @@ -545,6 +547,7 @@ def create_deploy_candidate(self, apps_to_update=None) -> "Optional[DeployCandid "dependencies": dependencies, "packages": packages, "environment_variables": environment_variables, + "mirror_server": self.mirror_server, } ).insert()