add base for npm engine

app/controllers/api/v1/release_engines/rubygems/compact_index_controller.rb

@@ -103,6 +103,7 @@ def to_versions(artifacts)
         CompactIndex::GemVersion.new(
           gemspec.version.to_s,
           gemspec.platform.to_s,
+          # FIXME(ezekg) add padding if base64 and missing (i.e. trailing =)
           artifact.checksum,
           nil, # will be calculated via versions file
           dependencies,

app/models/release_artifact.rb

@@ -455,7 +455,8 @@ class ReleaseArtifact < ApplicationRecord
 
   scope :gems, -> { for_filetype(:gem) }
 
-  def key = "artifacts/#{account_id}/#{release_id}/#{filename}"
+  def key_for(path) = "artifacts/#{account_id}/#{release_id}/#{path}"
+  def key           = key_for(filename)
 
   def presigner = Aws::S3::Presigner.new(client:)
 

app/models/release_engine.rb

@@ -12,6 +12,7 @@ class ReleaseEngine < ApplicationRecord
     tauri
     raw
     rubygems
+    npm
   ]
 
   has_many :packages,
@@ -110,6 +111,7 @@ def pypi?     = key == 'pypi'
   def tauri?    = key == 'tauri'
   def raw?      = key == 'raw'
   def rubygems? = key == 'rubygems'
+  def npm?      = key == 'npm'
 
   ##
   # deconstruct allows pattern pattern matching like:

app/models/release_package.rb

@@ -106,8 +106,9 @@ class ReleasePackage < ApplicationRecord
   scope :tauri,          ->     { for_engine_key('tauri') }
   scope :raw,            ->     { for_engine_key('raw') }
   scope :rubygems,       ->     { for_engine_key('rubygems') }
+  scope :npm,            ->     { for_engine_key('npm') }
 
-  delegate :pypi?, :tauri?, :raw?, :rubygems?,
+  delegate :pypi?, :tauri?, :raw?, :rubygems?, :npm?,
     to: :engine,
     allow_nil: true
 

app/workers/process_docker_image_worker.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+
+class ProcessDockerImageWorker < BaseWorker
+  MIN_TARBALL_SIZE  = 5.bytes    # to avoid processing empty or invalid tarballs
+  MAX_TARBALL_SIZE  = 1.gigabyte # to avoid downloading excessive tarballs
+  MAX_MANIFEST_SIZE = 1.megabyte # to avoid storing large manifests
+
+  sidekiq_options queue: :critical
+
+  def perform(artifact_id)
+    artifact = ReleaseArtifact.find(artifact_id)
+    return unless
+      artifact.processing?
+
+    # sanity check the image tarball
+    unless artifact.content_length.in?(MIN_TARBALL_SIZE..MAX_TARBALL_SIZE)
+      raise ImageNotAcceptableError, 'unacceptable filesize'
+    end
+
+    # download the image tarball
+    client = artifact.client
+    tgz    = client.get_object(bucket: artifact.bucket, key: artifact.key)
+                   .body
+
+    # gunzip and untar the image tarball
+    tar = gunzip(tgz)
+
+    untar tar do |archive|
+      archive.each do |entry|
+        case entry.full_name
+        in 'manifest.json'
+          raise ImageNotAcceptableError, 'manifest must be a manifest.json file' unless
+            entry.file?
+
+          raise ImageNotAcceptableError, 'manifest is too big' if
+            entry.size > MAX_MANIFEST_SIZE
+
+          # the manifest is already in json format
+          json = entry.read
+
+          ReleaseManifest.create!(
+            account_id: artifact.account_id,
+            environment_id: artifact.environment_id,
+            release_id: artifact.release_id,
+            release_artifact_id: artifact.id,
+            content: json,
+          )
+        in %r{^blobs/sha256/} if entry.file?
+          key = artifact.key_for(entry.full_name)
+
+          # skip if already uploaded
+          next if
+            client.head_object(bucket: artifact.bucket, key:) rescue false
+
+          # upload blob in chunks
+          client.put_object(bucket: artifact.bucket, key:) do |writer|
+            while (chunk = entry.read(16 * 1024)) # write in chunks
+              writer.write(chunk)
+            end
+          end
+        else
+        end
+      end
+    end
+
+    artifact.update!(status: 'UPLOADED')
+
+    BroadcastEventService.call(
+      event: 'artifact.upload.succeeded',
+      account: artifact.account,
+      resource: artifact,
+    )
+  rescue ImageNotAcceptableError,
+         Gem::Package::FormatError,
+         Zlib::GzipFile::Error,
+         Zlib::Error,
+         IOError => e
+    Keygen.logger.warn { "[workers.process-docker-image-worker] Error: #{e.class.name} - #{e.message}" }
+
+    artifact.update!(status: 'FAILED')
+
+    BroadcastEventService.call(
+      event: 'artifact.upload.failed',
+      account: artifact.account,
+      resource: artifact,
+    )
+  end
+
+  private
+
+  def gunzip(io)    = Zlib::GzipReader.new(io)
+  def untar(io, &)  = Gem::Package::TarReader.new(io, &)
+
+  class ImageNotAcceptableError < StandardError
+    def backtrace = nil # silence backtrace
+  end
+end

app/workers/process_npm_package_worker.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+
+class ProcessNpmPackageWorker < BaseWorker
+  MIN_TARBALL_SIZE  = 5.bytes      # to avoid processing empty or invalid tarballs
+  MAX_TARBALL_SIZE  = 25.megabytes # to avoid downloading large tarballs
+  MAX_MANIFEST_SIZE = 1.megabyte   # to avoid storing large manifests
+
+  sidekiq_options queue: :critical
+
+  def perform(artifact_id)
+    artifact = ReleaseArtifact.find(artifact_id)
+    return unless
+      artifact.processing?
+
+    # sanity check the package tarball
+    unless artifact.content_length.in?(MIN_TARBALL_SIZE..MAX_TARBALL_SIZE)
+      raise PackageNotAcceptableError, 'unacceptable filesize'
+    end
+
+    # download the package tarball
+    client = artifact.client
+    tgz    = client.get_object(bucket: artifact.bucket, key: artifact.key)
+                   .body
+
+    # gunzip and untar the package tarball
+    tar = gunzip(tgz)
+
+    untar tar do |archive|
+      # NOTE(ezekg) npm prefixes everything in the archive with package/
+      archive.seek('package/package.json') do |entry|
+        raise PackageNotAcceptableError, 'manifest must be a package.json file' unless
+          entry.file?
+
+        raise PackageNotAcceptableError, 'manifest is too big' if
+          entry.size > MAX_MANIFEST_SIZE
+
+        # the manifest is already in json format
+        json = entry.read
+
+        ReleaseManifest.create!(
+          account_id: artifact.account_id,
+          environment_id: artifact.environment_id,
+          release_id: artifact.release_id,
+          release_artifact_id: artifact.id,
+          content: json,
+        )
+      end
+    end
+
+    artifact.update!(status: 'UPLOADED')
+
+    BroadcastEventService.call(
+      event: 'artifact.upload.succeeded',
+      account: artifact.account,
+      resource: artifact,
+    )
+  rescue PackageNotAcceptableError,
+         Gem::Package::FormatError,
+         Zlib::GzipFile::Error,
+         Zlib::Error,
+         IOError => e
+    Keygen.logger.warn { "[workers.process-npm-package-worker] Error: #{e.class.name} - #{e.message}" }
+
+    artifact.update!(status: 'FAILED')
+
+    BroadcastEventService.call(
+      event: 'artifact.upload.failed',
+      account: artifact.account,
+      resource: artifact,
+    )
+  end
+
+  private
+
+  def gunzip(io)    = Zlib::GzipReader.new(io)
+  def untar(io, &)  = Gem::Package::TarReader.new(io, &)
+
+  class PackageNotAcceptableError < StandardError
+    def backtrace = nil # silence backtrace
+  end
+end

app/workers/wait_for_artifact_upload_worker.rb

@@ -46,6 +46,22 @@ def perform(artifact_id, enqueued_at = Time.current.iso8601)
       )
 
       ProcessRubyGemWorker.perform_async(artifact.id)
+    in filetype: ReleaseFiletype(:tar), engine: ReleaseEngine(:docker)
+      BroadcastEventService.call(
+        event: 'artifact.upload.processing',
+        account: artifact.account,
+        resource: artifact,
+      )
+
+      ProcessDockerImageWorker.perform_async(artifact.id)
+    in filetype: ReleaseFiletype(:tgz), engine: ReleaseEngine(:npm)
+      BroadcastEventService.call(
+        event: 'artifact.upload.processing',
+        account: artifact.account,
+        resource: artifact,
+      )
+
+      ProcessNpmPackageWorker.perform_async(artifact.id)
     else
       artifact.update!(status: 'UPLOADED')
 

spec/factories/release.rb

@@ -67,6 +67,10 @@
       package { build(:package, :rubygems, account:, product:, environment:) }
     end
 
+    trait :npm do
+      package { build(:package, :npm, account:, product:, environment:) }
+    end
+
     trait :stable do
       channel { build(:channel, :beta, account:) }
     end

spec/factories/release_artifact.rb

@@ -52,6 +52,24 @@
       arch     { nil }
     end
 
+    trait :npm_package do
+      release  { build(:release, :npm, account:, environment:) }
+      filename { "#{release.name.underscore.parameterize}-#{release.version}.tgz" }
+      filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 25.megabytes.to_i) }
+      filetype { build(:filetype, key: 'tgz', account:) }
+      platform { nil }
+      arch     { nil }
+    end
+
+    trait :docker_image do
+      release  { build(:release, :npm, account:, environment:) }
+      filename { "#{release.name.underscore.parameterize}.tar" }
+      filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 1.gigabyte.to_i) }
+      filetype { build(:filetype, key: 'tar', account:) }
+      platform { nil }
+      arch     { nil }
+    end
+
     trait :with_smanifest do
       after :create do |artifact|
         next if artifact.engine.nil?

spec/factories/release_engine.rb

@@ -27,5 +27,10 @@
       name { 'Rubygems' }
       key  { 'rubygems' }
     end
+
+    trait :npm do
+      name { 'npm' }
+      key  { 'npm' }
+    end
   end
 end

spec/factories/release_manifest.rb

@@ -21,5 +21,22 @@
         gemspec.to_yaml
       }
     end
+
+    trait :npm do
+      artifact { build(:artifact, :npm_package, account:, environment:) }
+      content  {
+        tgz = file_fixture('hello-2.0.0.tgz').read
+        tar = Zlib::GzipReader.new(tgz)
+        pkg = nil
+
+        Gem::Package::TarReader.new(tar) do |archive|
+          archive.seek('package/package.json') do |entry|
+            pkg = entry.read
+          end
+        end
+
+        pkg
+      }
+    end
   end
 end

spec/factories/release_package.rb

@@ -28,6 +28,10 @@
       engine { build(:engine, :rubygems, account:) }
     end
 
+    trait :npm do
+      engine { build(:engine, :npm, account:) }
+    end
+
     trait :licensed do
       product { build(:product, :licensed, account:, environment:) }
     end

spec/fixtures/packages/hello/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

spec/fixtures/packages/hello/README.md

@@ -0,0 +1 @@
+# Hello

spec/fixtures/packages/hello/package.json

@@ -0,0 +1,33 @@
+{
+  "version": "2.0.0",
+  "name": "hello",
+  "type": "module",
+  "description": "Hello world and beyond.",
+  "author": {
+    "email": "[email protected]",
+    "name": "hellohq"
+  },
+  "keywords": [
+    "hello",
+    "world"
+  ],
+  "homepage": "https://hello.example/",
+  "bugs": "https://github.example/hellohq/hello/issues",
+  "license": "MIT",
+  "files": [
+    "LICENSE",
+    "README.md",
+    "src/index.js"
+  ],
+  "main": "index.js",
+  "exports": {
+    ".": "./src/*"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.example/hellohq/hello"
+  },
+  "engines": {
+    "node": ">=0.22.0"
+  }
+}

spec/fixtures/packages/hello/src/index.js

@@ -0,0 +1,2 @@
+// hello greets provided subject
+export const hello = (subject = 'world') => `Hello, ${subject}!`