From d1217ceb434d9b6ec1598fd3d458f7e572cf43bf Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Fri, 13 Sep 2024 17:10:36 -0700 Subject: [PATCH] fwv --- .fixtures.yml | 8 ++++ README.md | 53 ++++++++++++++++++++++++ examples/instances.pp | 19 +++++++++ manifests/init.pp | 18 +++++++++ manifests/instance.pp | 74 ++++++++++++++++++++++++++++++++++ metadata.json | 35 ++++++++++++++++ spec/acceptance/init_spec.rb | 76 +++++++++++++++++++++++++++++++++++ spec/defines/instance_spec.rb | 46 +++++++++++++++++++++ 8 files changed, 329 insertions(+) create mode 100644 .fixtures.yml create mode 100644 README.md create mode 100644 examples/instances.pp create mode 100644 manifests/init.pp create mode 100644 manifests/instance.pp create mode 100644 metadata.json create mode 100644 spec/acceptance/init_spec.rb create mode 100644 spec/defines/instance_spec.rb diff --git a/.fixtures.yml b/.fixtures.yml new file mode 100644 index 0000000..28dc47d --- /dev/null +++ b/.fixtures.yml @@ -0,0 +1,8 @@ +# This file can be used to install module dependencies for unit testing +# See https://github.com/puppetlabs/puppetlabs_spec_helper#using-fixtures for details +--- +fixtures: + forge_modules: + stdlib: puppetlabs/stdlib + quadlets: puppet/quadlets + systemd: puppet/systemd diff --git a/README.md b/README.md new file mode 100644 index 0000000..ddaa3c3 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# s3daemon + +## Table of Contents + +1. [Overview](#overview) +1. [Description](#description) +1. [Usage - Configuration options and additional functionality](#usage) +1. [Reference - An under-the-hood peek at what the module is doing and how](#reference) + +## Overview + +[s3daemon](https://github.com/lsst-dm/s3daemon/) Client/server for pushing objects to S3 storage. + +## Description + +The server is intended to be able to maintain long-lived TCP connections, avoiding both authentication delays and TCP slow start on long bandwidth-delay product network segments. +Enabling multiple simultaneous parallel transfers also is intended to maximize usage of the network. + +The client is intended to allow "fire-and-forget" submissions of transfer requests by file-writing code. + +## Usage + +Example role defined via hiera. + +```yaml +--- +lookup_options: + s3daemon::instances: + merge: + strategy: deep +classes: + - s3daemon +s3daemon::instances: + foo: + s3_endpoint_url: https://s3.foo.example.com + aws_access_key_id: access_key_id + aws_secret_access_key: secret_access_key + port: 15556 + image: ghcr.io/lsst-dm/s3daemon:main + bar: + s3_endpoint_url: https://s3.bar.example.com + aws_access_key_id: access_key_id + aws_secret_access_key: secret_access_key + port: 15557 + image: ghcr.io/lsst-dm/s3daemon:sha-b5e72fa + volumes: + - "/home:/home" + - "/opt:/opt" +``` + +## Reference + +See [REFERENCE](REFERENCE.md) diff --git a/examples/instances.pp b/examples/instances.pp new file mode 100644 index 0000000..300c649 --- /dev/null +++ b/examples/instances.pp @@ -0,0 +1,19 @@ +class { 's3daemon': + instances => { + 'foo' => { + 's3_endpoint_url' => 'https://s3.foo.example.com', + 'aws_access_key_id' => 'access_key_id', + 'aws_secret_access_key' => 'secret_access_key', + 'port' => 15556, + 'image' => 'ghcr.io/lsst-dm/s3daemon:main', + }, + 'bar' => { + 's3_endpoint_url' => 'https://s3.bar.example.com', + 'aws_access_key_id' => 'access_key_id', + 'aws_secret_access_key' => 'secret_access_key', + 'port' => 15557, + 'image' => 'ghcr.io/lsst-dm/s3daemon:sha-b5e72fa', + 'volumes' => ['/home:/home', '/opt:/opt'], + }, + }, +} diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 0000000..cd66b11 --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,18 @@ +# +# @summary Client/server for pushing objects to S3 storage. +# +# @param instances +# A hash of instances to configure. The key is the instance name and the value +# is a hash of `s3daemon::instance` parameters. +# +class s3daemon ( + Optional[Hash[String[1], Hash]] $instances = undef +) { + if $instances != undef { + $instances.each |$name, $params| { + s3daemon::instance { $name: + * => $params, + } + } + } +} diff --git a/manifests/instance.pp b/manifests/instance.pp new file mode 100644 index 0000000..fa79a30 --- /dev/null +++ b/manifests/instance.pp @@ -0,0 +1,74 @@ +# @summary +# Deploy the s3daemon service +# +# @param s3_endpoint_url +# The URL of the S3 endpoint to which the s3daemon service will send files. +# +# @param aws_access_key_id +# The AWS access key ID to use for authentication. +# +# @param aws_secret_access_key +# The AWS secret access key to use for authentication. +# +# @param port +# The tcp port on which the s3daemon service will listen. +# Default: 16666 +# +# @param image +# The container image to use for the s3daemon service. +# +# @param volumes +# An array of volumes to mount in the container. Uses the format +# '/host:/contaner'. E.g. ['/home:/home', '/data:/data'] +# +# Default: ['/home:/home'] +# +define s3daemon::instance ( + Stdlib::HTTPUrl $s3_endpoint_url, + Variant[String[1], Sensitive[String[1]]] $aws_access_key_id, + Variant[String[1], Sensitive[String[1]]] $aws_secret_access_key, + Stdlib::Port $port = 15556, + String[1] $image = 'ghcr.io/lsst-dm/s3daemon:main', + Array[Stdlib::Absolutepath] $volumes = ['/home:/home'], +) { + file { "/etc/sysconfig/s3daemon-${name}": + ensure => file, + show_diff => false, # don't leak secrets in the logs + mode => '0600', # only root should be able to read the secrets + # lint:ignore:strict_indent + content => @("CONTENT"), + S3DAEMON_PORT=${port} + S3_ENDPOINT_URL=${s3_endpoint_url} + AWS_ACCESS_KEY_ID=${aws_access_key_id.unwrap} + AWS_SECRET_ACCESS_KEY=${aws_secret_access_key.unwrap} + | CONTENT + # lint:endignore + } + + $nobody = 65534 + quadlets::quadlet { "s3daemon-${name}.container": + ensure => present, + active => true, + unit_entry => { + 'Description' => 's3 file sending service', + 'Requires' => 'network-online.target', + 'After' => 'network-online.target', + }, + container_entry => { + 'EnvironmentFile' => [ + "/etc/sysconfig/s3daemon-${name}", + ], + 'Image' => $image, + 'Network' => 'host', + 'Volume' => $volumes, + 'User' => $nobody, + 'Group' => $nobody, + }, + service_entry => { + 'Restart' => 'always', + }, + install_entry => { + 'WantedBy' => 'default.target', + }, + } +} diff --git a/metadata.json b/metadata.json new file mode 100644 index 0000000..c7c15ff --- /dev/null +++ b/metadata.json @@ -0,0 +1,35 @@ +{ + "name": "lsst-s3daemon", + "version": "1.0.0-rc0", + "author": "AURA/LSST/Rubin Observatory", + "summary": "Client/server for pushing objects to S3 storage.", + "license": "Apache-2.0", + "source": "https://github.com/lsst-it/puppet-s3daemon.git", + "tags": [ + "s3" + ], + "dependencies": [ + { + "name": "puppetlabs/stdlib", + "version_requirement": ">= 9.0.0 < 10.0.0" + }, + { + "name": "puppet/quadlets", + "version_requirement": ">= 1.0.0 < 2.0.0" + } + ], + "operatingsystem_support": [ + { + "operatingsystem": "AlmaLinux", + "operatingsystemrelease": [ + "9" + ] + } + ], + "requirements": [ + { + "name": "puppet", + "version_requirement": ">= 7.0.0 < 9.0.0" + } + ] +} diff --git a/spec/acceptance/init_spec.rb b/spec/acceptance/init_spec.rb new file mode 100644 index 0000000..6c54af8 --- /dev/null +++ b/spec/acceptance/init_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'spec_helper_acceptance' + +describe 's3daemon' do + shell('dnf install -y podman-docker') # used by serverspec + + context 'with instances param' do + include_examples 'the example', 'instances.pp' + + it_behaves_like 'an idempotent resource' + + context 'instance foo' do + describe file('/etc/sysconfig/s3daemon-foo') do + it { is_expected.to be_file } + it { is_expected.to be_mode '600' } + it { is_expected.to contain 'S3DAEMON_PORT=15556' } + it { is_expected.to contain 'S3_ENDPOINT_URL=https://s3.foo.example.com' } + it { is_expected.to contain 'AWS_ACCESS_KEY_ID=access_key_id' } + it { is_expected.to contain 'AWS_SECRET_ACCESS_KEY=secret_access_key' } + end + + describe docker_container('systemd-s3daemon-foo') do + its(['Config.Image']) { is_expected.to eq 'ghcr.io/lsst-dm/s3daemon:main' } + its(['Config.User']) { is_expected.to eq '65534:65534' } + its(['HostConfig.NetworkMode']) { is_expected.to eq 'host' } + + its(['Mounts']) do + is_expected.to match([include('Source' => '/home')]) + end + end + + describe service('s3daemon-foo') do + it { is_expected.to be_running } + it { is_expected.to be_enabled } + end + + describe port(15_556) do + it { is_expected.to be_listening.with('tcp') } + end + end + + context 'instance bar' do + describe file('/etc/sysconfig/s3daemon-bar') do + it { is_expected.to be_file } + it { is_expected.to be_mode '600' } + it { is_expected.to contain 'S3DAEMON_PORT=15557' } + it { is_expected.to contain 'S3_ENDPOINT_URL=https://s3.bar.example.com' } + it { is_expected.to contain 'AWS_ACCESS_KEY_ID=access_key_id' } + it { is_expected.to contain 'AWS_SECRET_ACCESS_KEY=secret_access_key' } + end + + describe docker_container('systemd-s3daemon-bar') do + its(['Config.Image']) { is_expected.to eq 'ghcr.io/lsst-dm/s3daemon:sha-b5e72fa' } + its(['Config.User']) { is_expected.to eq '65534:65534' } + its(['HostConfig.NetworkMode']) { is_expected.to eq 'host' } + + its(['Mounts']) do + is_expected.to match([ + include('Source' => '/home'), + include('Source' => '/opt'), + ]) + end + end + + describe service('s3daemon-bar') do + it { is_expected.to be_running } + it { is_expected.to be_enabled } + end + + describe port(15_557) do + it { is_expected.to be_listening.with('tcp') } + end + end + end +end diff --git a/spec/defines/instance_spec.rb b/spec/defines/instance_spec.rb new file mode 100644 index 0000000..0301968 --- /dev/null +++ b/spec/defines/instance_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe 's3daemon::instance' do + on_supported_os.each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:title) { 'foo' } + let(:params) do + { + s3_endpoint_url: 'https://s3.example.com', + aws_access_key_id: 'foo', + aws_secret_access_key: 'bar', + } + end + + it { is_expected.to compile.with_all_deps } + + it do + is_expected.to contain_file("/etc/sysconfig/s3daemon-#{title}").with( + ensure: 'file', + show_diff: false, + mode: '0600', + content: %r{S3DAEMON_PORT=15556} + ) + end + + it do + is_expected.to contain_quadlets__quadlet("s3daemon-#{title}.container").with( + container_entry: { + 'EnvironmentFile' => ["/etc/sysconfig/s3daemon-#{title}"], + 'Image' => 'ghcr.io/lsst-dm/s3daemon:main', + 'Network' => 'host', + 'Volume' => [ + '/home:/home', + # '/data:/data', + ], + 'User' => 65_534, + 'Group' => 65_534, + } + ) + end + end + end +end