fwv

.fixtures.yml

@@ -0,0 +1,8 @@
+# This file can be used to install module dependencies for unit testing
+# See https://github.com/puppetlabs/puppetlabs_spec_helper#using-fixtures for details
+---
+fixtures:
+ forge_modules:
+ stdlib: puppetlabs/stdlib
+ quadlets: puppet/quadlets
+ systemd: puppet/systemd

README.md

@@ -0,0 +1,53 @@
+# s3daemon
+
+## Table of Contents
+
+1. [Overview](#overview)
+1. [Description](#description)
+1. [Usage - Configuration options and additional functionality](#usage)
+1. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
+
+## Overview
+
+[s3daemon](https://github.com/lsst-dm/s3daemon/) Client/server for pushing objects to S3 storage.
+
+## Description
+
+The server is intended to be able to maintain long-lived TCP connections, avoiding both authentication delays and TCP slow start on long bandwidth-delay product network segments.
+Enabling multiple simultaneous parallel transfers also is intended to maximize usage of the network.
+
+The client is intended to allow "fire-and-forget" submissions of transfer requests by file-writing code.
+
+## Usage
+
+Example role defined via hiera.
+
+```yaml
+---
+lookup_options:
+ s3daemon::instances:
+ merge:
+ strategy: deep
+classes:
+ - s3daemon
+s3daemon::instances:
+ foo:
+ s3_endpoint_url: https://s3.foo.example.com
+ aws_access_key_id: access_key_id
+ aws_secret_access_key: secret_access_key
+ port: 15556
+ image: ghcr.io/lsst-dm/s3daemon:main
+ bar:
+ s3_endpoint_url: https://s3.bar.example.com
+ aws_access_key_id: access_key_id
+ aws_secret_access_key: secret_access_key
+ port: 15557
+ image: ghcr.io/lsst-dm/s3daemon:sha-b5e72fa
+ volumes:
+ - "/home:/home"
+ - "/opt:/opt"
+```
+
+## Reference
+
+See [REFERENCE](REFERENCE.md)

examples/instances.pp

@@ -0,0 +1,19 @@
+class { 's3daemon':
+ instances => {
+ 'foo' => {
+ 's3_endpoint_url' => 'https://s3.foo.example.com',
+ 'aws_access_key_id' => 'access_key_id',
+ 'aws_secret_access_key' => 'secret_access_key',
+ 'port' => 15556,
+ 'image' => 'ghcr.io/lsst-dm/s3daemon:main',
+ },
+ 'bar' => {
+ 's3_endpoint_url' => 'https://s3.bar.example.com',
+ 'aws_access_key_id' => 'access_key_id',
+ 'aws_secret_access_key' => 'secret_access_key',
+ 'port' => 15557,
+ 'image' => 'ghcr.io/lsst-dm/s3daemon:sha-b5e72fa',
+ 'volumes' => ['/home:/home', '/opt:/opt'],
+ },
+ },
+}

manifests/init.pp

@@ -0,0 +1,18 @@
+#
+# @summary Client/server for pushing objects to S3 storage.
+#
+# @param instances
+# A hash of instances to configure. The key is the instance name and the value
+# is a hash of `s3daemon::instance` parameters.
+#
+class s3daemon (
+ Optional[Hash[String[1], Hash]] $instances = undef
+) {
+ if $instances != undef {
+ $instances.each |$name, $params| {
+ s3daemon::instance { $name:
+ * => $params,
+ }
+ }
+ }
+}

manifests/instance.pp

@@ -0,0 +1,74 @@
+# @summary
+# Deploy the s3daemon service
+#
+# @param s3_endpoint_url
+# The URL of the S3 endpoint to which the s3daemon service will send files.
+#
+# @param aws_access_key_id
+# The AWS access key ID to use for authentication.
+#
+# @param aws_secret_access_key
+# The AWS secret access key to use for authentication.
+#
+# @param port
+# The tcp port on which the s3daemon service will listen.
+# Default: 16666
+#
+# @param image
+# The container image to use for the s3daemon service.
+#
+# @param volumes
+# An array of volumes to mount in the container. Uses the format
+# '/host:/contaner'. E.g. ['/home:/home', '/data:/data']
+#
+# Default: ['/home:/home']
+#
+define s3daemon::instance (
+ Stdlib::HTTPUrl $s3_endpoint_url,
+ Variant[String[1], Sensitive[String[1]]] $aws_access_key_id,
+ Variant[String[1], Sensitive[String[1]]] $aws_secret_access_key,
+ Stdlib::Port $port = 15556,
+ String[1] $image = 'ghcr.io/lsst-dm/s3daemon:main',
+ Array[Stdlib::Absolutepath] $volumes = ['/home:/home'],
+) {
+ file { "/etc/sysconfig/s3daemon-${name}":
+ ensure => file,
+ show_diff => false, # don't leak secrets in the logs
+ mode => '0600', # only root should be able to read the secrets
+ # lint:ignore:strict_indent
+ content => @("CONTENT"),
+ S3DAEMON_PORT=${port}
+ S3_ENDPOINT_URL=${s3_endpoint_url}
+ AWS_ACCESS_KEY_ID=${aws_access_key_id.unwrap}
+ AWS_SECRET_ACCESS_KEY=${aws_secret_access_key.unwrap}
+ | CONTENT
+ # lint:endignore
+ }
+
+ $nobody = 65534
+ quadlets::quadlet { "s3daemon-${name}.container":
+ ensure => present,
+ active => true,
+ unit_entry => {
+ 'Description' => 's3 file sending service',
+ 'Requires' => 'network-online.target',
+ 'After' => 'network-online.target',
+ },
+ container_entry => {
+ 'EnvironmentFile' => [
+ "/etc/sysconfig/s3daemon-${name}",
+ ],
+ 'Image' => $image,
+ 'Network' => 'host',
+ 'Volume' => $volumes,
+ 'User' => $nobody,
+ 'Group' => $nobody,
+ },
+ service_entry => {
+ 'Restart' => 'always',
+ },
+ install_entry => {
+ 'WantedBy' => 'default.target',
+ },
+ }
+}

metadata.json

@@ -0,0 +1,35 @@
+{
+ "name": "lsst-s3daemon",
+ "version": "1.0.0-rc0",
+ "author": "AURA/LSST/Rubin Observatory",
+ "summary": "Client/server for pushing objects to S3 storage.",
+ "license": "Apache-2.0",
+ "source": "https://github.com/lsst-it/puppet-s3daemon.git",
+ "tags": [
+ "s3"
+ ],
+ "dependencies": [
+ {
+ "name": "puppetlabs/stdlib",
+ "version_requirement": ">= 9.0.0 < 10.0.0"
+ },
+ {
+ "name": "puppet/quadlets",
+ "version_requirement": ">= 1.0.0 < 2.0.0"
+ }
+ ],
+ "operatingsystem_support": [
+ {
+ "operatingsystem": "AlmaLinux",
+ "operatingsystemrelease": [
+ "9"
+ ]
+ }
+ ],
+ "requirements": [
+ {
+ "name": "puppet",
+ "version_requirement": ">= 7.0.0 < 9.0.0"
+ }
+ ]
+}

spec/acceptance/init_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'spec_helper_acceptance'
+
+describe 's3daemon' do
+ shell('dnf install -y podman-docker') # used by serverspec
+
+ context 'with instances param' do
+ include_examples 'the example', 'instances.pp'
+
+ it_behaves_like 'an idempotent resource'
+
+ context 'instance foo' do
+ describe file('/etc/sysconfig/s3daemon-foo') do
+ it { is_expected.to be_file }
+ it { is_expected.to be_mode '600' }
+ it { is_expected.to contain 'S3DAEMON_PORT=15556' }
+ it { is_expected.to contain 'S3_ENDPOINT_URL=https://s3.foo.example.com' }
+ it { is_expected.to contain 'AWS_ACCESS_KEY_ID=access_key_id' }
+ it { is_expected.to contain 'AWS_SECRET_ACCESS_KEY=secret_access_key' }
+ end
+
+ describe docker_container('systemd-s3daemon-foo') do
+ its(['Config.Image']) { is_expected.to eq 'ghcr.io/lsst-dm/s3daemon:main' }
+ its(['Config.User']) { is_expected.to eq '65534:65534' }
+ its(['HostConfig.NetworkMode']) { is_expected.to eq 'host' }
+
+ its(['Mounts']) do
+ is_expected.to match([include('Source' => '/home')])
+ end
+ end
+
+ describe service('s3daemon-foo') do
+ it { is_expected.to be_running }
+ it { is_expected.to be_enabled }
+ end
+
+ describe port(15_556) do
+ it { is_expected.to be_listening.with('tcp') }
+ end
+ end
+
+ context 'instance bar' do
+ describe file('/etc/sysconfig/s3daemon-bar') do
+ it { is_expected.to be_file }
+ it { is_expected.to be_mode '600' }
+ it { is_expected.to contain 'S3DAEMON_PORT=15557' }
+ it { is_expected.to contain 'S3_ENDPOINT_URL=https://s3.bar.example.com' }
+ it { is_expected.to contain 'AWS_ACCESS_KEY_ID=access_key_id' }
+ it { is_expected.to contain 'AWS_SECRET_ACCESS_KEY=secret_access_key' }
+ end
+
+ describe docker_container('systemd-s3daemon-bar') do
+ its(['Config.Image']) { is_expected.to eq 'ghcr.io/lsst-dm/s3daemon:sha-b5e72fa' }
+ its(['Config.User']) { is_expected.to eq '65534:65534' }
+ its(['HostConfig.NetworkMode']) { is_expected.to eq 'host' }
+
+ its(['Mounts']) do
+ is_expected.to match([
+ include('Source' => '/home'),
+ include('Source' => '/opt'),
+ ])
+ end
+ end
+
+ describe service('s3daemon-bar') do
+ it { is_expected.to be_running }
+ it { is_expected.to be_enabled }
+ end
+
+ describe port(15_557) do
+ it { is_expected.to be_listening.with('tcp') }
+ end
+ end
+ end
+end

spec/defines/instance_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 's3daemon::instance' do
+ on_supported_os.each do |os, os_facts|
+ context "on #{os}" do
+ let(:facts) { os_facts }
+ let(:title) { 'foo' }
+ let(:params) do
+ {
+ s3_endpoint_url: 'https://s3.example.com',
+ aws_access_key_id: 'foo',
+ aws_secret_access_key: 'bar',
+ }
+ end
+
+ it { is_expected.to compile.with_all_deps }
+
+ it do
+ is_expected.to contain_file("/etc/sysconfig/s3daemon-#{title}").with(
+ ensure: 'file',
+ show_diff: false,
+ mode: '0600',
+ content: %r{S3DAEMON_PORT=15556}
+ )
+ end
+
+ it do
+ is_expected.to contain_quadlets__quadlet("s3daemon-#{title}.container").with(
+ container_entry: {
+ 'EnvironmentFile' => ["/etc/sysconfig/s3daemon-#{title}"],
+ 'Image' => 'ghcr.io/lsst-dm/s3daemon:main',
+ 'Network' => 'host',
+ 'Volume' => [
+ '/home:/home',
+ # '/data:/data',
+ ],
+ 'User' => 65_534,
+ 'Group' => 65_534,
+ }
+ )
+ end
+ end
+ end
+end