Use mini_portile2 to build libmagic

.travis.yml

@@ -1,5 +1,5 @@
 language: ruby
-dist: xenial
+dist: bionic
 
 cache:
  bundler: true
@@ -18,6 +18,10 @@ env:
  matrix:
  - COMPILER=gcc
  - COMPILER=clang
+ - COMPILER=gcc RAKE_COMPILER_OPTIONS="-- --disable-static"
+ - COMPILER=clang RAKE_COMPILER_OPTIONS="-- --disable-static"
+ - COMPILER=gcc RAKE_COMPILER_OPTIONS="-- --use-system-libraries"
+ - COMPILER=clang RAKE_COMPILER_OPTIONS="-- --use-system-libraries"
 
 matrix:
  fast_finish: true
@@ -38,12 +42,13 @@ before_install:
  - bundle config set --local path 'vendor/bundle'
  - bundle config set --local without 'development'
  - export CC="$(which $COMPILER)"
+ - if [[ $RAKE_COMPILER_OPTIONS =~ use-system-libraries ]]; then sudo apt update && sudo apt install libmagic1 libmagic-dev; fi
 
 before_script:
  - export CFLAGS="-I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib"
  - export LD_LIBRARY_PATH="${PREFIX}/lib"
  - bundle exec rake clean
 
 script:
- - bundle exec rake compile
+ - bundle exec rake compile $RAKE_COMPILER_OPTIONS
  - bundle exec rake test

Rakefile

@@ -11,7 +11,11 @@ CLEAN.include FileList['**/*{.o,.so,.bundle,.log}'],
  FileList['**/Makefile']
 
 CLOBBER.include FileList['lib/**/*.so'],
- FileList['doc/**/*']
+ FileList['doc/**/*'],
+ FileList['tmp/'],
+ FileList['ext/magic/extconf.h'],
+ FileList['ext/magic/tmp'],
+ FileList['ports/']
 
 gem = eval File.read('ruby-magic.gemspec')
 

ext/magic/extconf.rb

@@ -1,42 +1,152 @@
 # frozen_string_literal: true
 
+require 'find'
 require 'mkmf'
-require 'digest'
-require 'open-uri'
-
-LIBMAGIC_TAG = '5.39'.freeze
-
-workdir = Dir.pwd
-libdir = File.join(workdir, 'file-' + LIBMAGIC_TAG)
-gemdir = File.expand_path(File.join(__dir__, '../..'))
-gem_ext_dir = File.join(gemdir, 'lib', 'ext')
-gem_include_dir = File.join(gem_ext_dir, 'include')
-gem_lib_dir = File.join(gem_ext_dir, 'lib')
-build_lib_dir = File.join(libdir, 'src', '.libs')
-
-expected_sha256 = 'f05d286a76d9556243d0cb05814929c2ecf3a5ba07963f8f70bfaaa70517fad1'
-filename = "#{workdir}/file.tar.gz"
-
-unless File.exist?(filename)
- File.open(filename, 'wb') do |target_file|
- URI.open("https://fossies.org/linux/misc/file-#{LIBMAGIC_TAG}.tar.gz", "User-Agent" => "RubyMagic/#{RUBY_DESCRIPTION}") do |read_file|
- target_file.write(read_file.read)
+require 'pathname'
+
+LIBMAGIC_TAG = '5.39'
+LIBIMAGE_SHA256 = 'f05d286a76d9556243d0cb05814929c2ecf3a5ba07963f8f70bfaaa70517fad1'
+
+# helpful constants
+PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+# The gem version constraint in the Rakefile is not respected at install time.
+# Keep this version in sync with the one in the Rakefile !
+REQUIRED_MINI_PORTILE_VERSION = "~> 2.5.0"
+
+MAGIC_HELP_MESSAGE = <<~HELP
+ USAGE: ruby #{$0} [options]
+
+ Flags that are always valid:
+
+ --use-system-libraries
+ --enable-system-libraries
+ Use system libraries instead of building and using the packaged libraries.
+
+ --disable-system-libraries
+ Use the packaged libraries, and ignore the system libraries. This is the default on most
+ platforms, and overrides `--use-system-libraries` and the environment variable
+ `RB_MAGIC_USE_SYSTEM_LIBRARIES`.
+
+ --disable-clean
+ Do not clean out intermediate files after successful build.
+
+ Flags only used when building and using the packaged libraries:
+
+ --disable-static
+ Do not statically link packaged libraries, instead use shared libraries.
+
+ --enable-cross-build
+ Enable cross-build mode. (You probably do not want to set this manually.)
+
+ Flags only used when using system libraries:
+
+ Related to libmagic:
+
+ --with-magic-dir=DIRECTORY
+ Look for libmagic headers and library in DIRECTORY.
+
+ --with-magic-lib=DIRECTORY
+ Look for libmagic library in DIRECTORY.
+
+ --with-magic-include=DIRECTORY
+ Look for libmagic headers in DIRECTORY.
+
+ Environment variables used:
+
+ CC
+ Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']`
+
+ CPPFLAGS
+ If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor
+
+ CFLAGS
+ If this string is accepted by the compiler, add it to the flags passed to the compiler
+
+ LDFLAGS
+ If this string is accepted by the linker, add it to the flags passed to the linker
+
+ LIBS
+ Add this string to the flags passed to the linker
+HELP
+
+def process_recipe(name, version, static_p, cross_p)
+ require 'rubygems'
+ gem('mini_portile2', REQUIRED_MINI_PORTILE_VERSION)
+ require 'mini_portile2'
+ message("Using mini_portile version #{MiniPortile::VERSION}\n")
+
+ MiniPortile.new(name, version).tap do |recipe|
+ # Prefer host_alias over host in order to use i586-mingw32msvc as
+ # correct compiler prefix for cross build, but use host if not set.
+ recipe.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"]
+ recipe.target = File.join(PACKAGE_ROOT_DIR, "ports")
+ recipe.configure_options << "--libdir=#{File.join(recipe.path, 'lib')}"
+
+ yield recipe
+
+ env = Hash.new do |hash, key|
+ hash[key] = (ENV[key]).to_s
+ end
+
+ recipe.configure_options.flatten!
+
+ recipe.configure_options = [
+ "--disable-silent-rules",
+ "--disable-dependency-tracking",
+ "--enable-fsect-man5"
+ ]
+
+ if static_p
+ recipe.configure_options += [
+ "--disable-shared",
+ "--enable-static",
+ ]
+ env["CFLAGS"] = concat_flags(env["CFLAGS"], "-fPIC")
+ else
+ recipe.configure_options += [
+ "--enable-shared",
+ "--disable-static",
+ ]
+ end
+
+ if cross_p
+ recipe.configure_options += [
+ "--target=#{recipe.host}",
+ "--host=#{recipe.host}",
+ ]
  end
- end
 
- checksum = Digest::SHA256.hexdigest(File.read(filename))
+ recipe.configure_options += env.map do |key, value|
+ "#{key}=#{value.strip}"
+ end
 
- if checksum != expected_sha256
- raise "SHA256 of #{filename} does not match: got #{checksum}, expected #{expected_sha256}"
+  recipe.cook
+ recipe.activate
  end
 end
 
-system("tar -xzf #{filename}") || raise('ERROR')
-system("cd #{libdir} && ./configure --prefix=#{gem_ext_dir} && make install") || raise('ERROR')
+#
+# utility functions
+#
+def config_clean?
+ enable_config('clean', true)
+end
+
+def config_static?
+ default_static = !truffle?
+ enable_config("static", default_static)
+end
+
+def config_cross_build?
+ enable_config("cross-build")
+end
 
-$LOCAL_LIBS << '-lmagic'
-$LIBPATH << gem_lib_dir
-$CFLAGS << " -I #{libdir}/src"
+def config_system_libraries?
+ enable_config("system-libraries", ENV.key?("RB_MAGIC_USE_SYSTEM_LIBRARIES")) do |_, default|
+ arg_config('--use-system-libraries', default)
+ end
+end
 
 def darwin?
  RbConfig::CONFIG['target_os'] =~ /darwin/
@@ -46,6 +156,89 @@ def windows?
  RbConfig::CONFIG['target_os'] =~ /mswin|mingw32|windows/
 end
 
+def truffle?
+ ::RUBY_ENGINE == 'truffleruby'
+end
+
+def concat_flags(*args)
+ args.compact.join(" ")
+end
+
+def do_help
+ print(MAGIC_HELP_MESSAGE)
+ exit!(0)
+end
+
+def do_clean
+ root = Pathname(PACKAGE_ROOT_DIR)
+ pwd = Pathname(Dir.pwd)
+
+ # Skip if this is a development work tree
+ unless (root + '.git').exist?
+ message("Cleaning files only used during build.\n")
+
+ # (root + 'tmp') cannot be removed at this stage because
+ # libmagic.so is yet to be copied to lib.
+
+ # clean the ports build directory
+ Pathname.glob(pwd.join('tmp', '*', 'ports')) do |dir|
+ FileUtils.rm_rf(dir, verbose: true)
+ end
+
+ FileUtils.rm_rf(root + 'ports' + 'archives', verbose: true)
+
+ # Remove everything but share/ directory
+ remove_paths = %w[bin include]
+ remove_paths << 'lib' if config_static?
+
+ Pathname.glob(File.join(root, 'ports', '*', 'libmagic', '*')) do |dir|
+ remove_paths.each do |path|
+ remove_dir = File.join(dir, path)
+ FileUtils.rm_rf(remove_dir, verbose: true)
+ end
+ end
+ end
+
+ exit!(0)
+end
+
+#
+# main
+#
+do_help if arg_config('--help')
+do_clean if arg_config('--clean')
+
+if config_system_libraries?
+ message "Building ruby-magic using system libraries.\n"
+
+ dir_config('magic')
+else
+ message "Building ruby-magic using packaged libraries.\n"
+
+ static_p = config_static?
+ message "Static linking is #{static_p ? 'enabled' : 'disabled'}.\n"
+ cross_build_p = config_cross_build?
+ message "Cross build is #{cross_build_p ? 'enabled' : 'disabled'}.\n"
+
+ libmagic_recipe = process_recipe('libmagic', LIBMAGIC_TAG, static_p, cross_build_p) do |recipe|
+ recipe.files = [{
+ url: "https://ruby-magic.s3.eu-central-1.amazonaws.com/file-#{recipe.version}.tar.gz",
+ sha256: LIBIMAGE_SHA256
+ }]
+ end
+
+ $LIBPATH = [File.join(libmagic_recipe.path, 'lib')]
+ $CFLAGS << " -I#{File.join(libmagic_recipe.path, 'include')} "
+ $LDFLAGS += " -Wl,-rpath,#{libmagic_recipe.path}/lib"
+
+ if static_p
+ ENV['PKG_CONFIG_PATH'] = "#{libmagic_recipe.path}/lib/pkgconfig"
+ # mkmf appends -- to the first option
+ $LIBS += " " + pkg_config('libmagic', 'libs --static')
+ $LIBS += " " + File.join(libmagic_recipe.path, 'lib', "libmagic.#{$LIBEXT}")
+ end
+end
+
 if ENV['CC']
  RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC']
 end
@@ -188,7 +381,17 @@ def windows?
  have_func(f)
 end
 
-dir_config('magic', [gem_include_dir], [gem_lib_dir])
-
 create_header
 create_makefile('magic/magic')
+
+if config_clean?
+ # Do not clean if run in a development work tree.
+ File.open('Makefile', 'at') do |mk|
+ mk.print(<<~EOF)
+
+ all: clean-ports
+ clean-ports: $(DLLIB)
+ \t-$(Q)$(RUBY) $(srcdir)/extconf.rb --clean --#{static_p ? 'enable' : 'disable'}-static
+ EOF
+ end
+end

ruby-magic.gemspec

@@ -49,4 +49,6 @@ Gem::Specification.new do |s|
 
  s.cert_chain = [ 'kwilczynski-public.pem' ]
  s.signing_key = signing_key if File.exist?(signing_key)
+
+ s.add_runtime_dependency("mini_portile2", "~> 2.5.0") # keep version in sync with extconf.rb
 end

test/test_magic.rb

@@ -55,7 +55,13 @@ def test_magic_new_instance
  end
 
  def test_magic_new_instance_default_flags
- assert_equal(0, @magic.flags)
+ # If libmagic needs to search more than one path, it will enable the CHECK flag.
+ # See https://github.com/kwilczynski/ruby-magic/pull/5#issuecomment-808686480
+ if @magic.paths.length > 1
+ assert_equal(Magic::CHECK, @magic.flags)
+ else
+ assert_equal(Magic::NONE, @magic.flags)
+ end
  end
 
  def test_magic_new_with_block