diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff04826..64b71a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,9 @@ name: re2 Tests + concurrency: group: "${{github.workflow}}-${{github.sha}}" cancel-in-progress: true + on: workflow_dispatch: push: @@ -9,6 +11,9 @@ on: - main pull_request: +env: + RUBYOPT: "--enable-frozen-string-literal --debug-frozen-string-literal --debug" + jobs: build-cruby-gem: name: "Build CRuby gem" diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 9d6abb3..802218b 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -11,370 +11,324 @@ require 'mkmf' require_relative 'recipes' -RE2_HELP_MESSAGE = <<~HELP - USAGE: ruby #{$0} [options] +module RE2 + class Extconf + def configure + configure_cross_compiler - Flags that are always valid: + if config_system_libraries? + build_with_system_libraries + else + build_with_vendored_libraries + end - --enable-system-libraries - Use system libraries instead of building and using the packaged libraries. + build_extension - --disable-system-libraries - Use the packaged libraries, and ignore the system libraries. This is the default. + create_makefile("re2") + end + def print_help + print(<<~TEXT) + USAGE: ruby #{$0} [options] - Flags only used when using system libraries: + Flags that are always valid: - Related to re2 library: + --enable-system-libraries + Use system libraries instead of building and using the packaged libraries. - --with-re2-dir=DIRECTORY - Look for re2 headers and library in DIRECTORY. + --disable-system-libraries + Use the packaged libraries, and ignore the system libraries. This is the default. - Flags only used when building and using the packaged libraries: + Flags only used when using system libraries: - --enable-cross-build - Enable cross-build mode. (You probably do not want to set this manually.) + Related to re2 library: + --with-re2-dir=DIRECTORY + Look for re2 headers and library in DIRECTORY. - Environment variables used: - CC - Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']` + Flags only used when building and using the packaged libraries: - CPPFLAGS - If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor + --enable-cross-build + Enable cross-build mode. (You probably do not want to set this manually.) - CFLAGS - If this string is accepted by the compiler, add it to the flags passed to the compiler - LDFLAGS - If this string is accepted by the linker, add it to the flags passed to the linker + Environment variables used: - LIBS - Add this string to the flags passed to the linker -HELP + CC + Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']` -# -# utility functions -# -def config_system_libraries? - enable_config("system-libraries", ENV.key?('RE2_USE_SYSTEM_LIBRARIES')) -end + CPPFLAGS + If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor -def config_cross_build? - enable_config("cross-build") -end + CFLAGS + If this string is accepted by the compiler, add it to the flags passed to the compiler -def concat_flags(*args) - args.compact.join(" ") -end + LDFLAGS + If this string is accepted by the linker, add it to the flags passed to the linker -def do_help - print(RE2_HELP_MESSAGE) - exit!(0) -end + LIBS + Add this string to the flags passed to the linker + TEXT + end -def darwin? - RbConfig::CONFIG["target_os"].include?("darwin") -end + private -def windows? - RbConfig::CONFIG["target_os"].match?(/mingw|mswin/) -end + def configure_cross_compiler + RbConfig::CONFIG["CC"] = RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] if ENV["CC"] + RbConfig::CONFIG["CXX"] = RbConfig::MAKEFILE_CONFIG["CXX"] = ENV["CXX"] if ENV["CXX"] + end -def freebsd? - RbConfig::CONFIG["target_os"].include?("freebsd") -end + def build_with_system_libraries + header_dirs = [ + "/usr/local/include", + "/opt/homebrew/include", + "/usr/include" + ] -def target_host - # We use 'host' to set compiler prefix for cross-compiling. Prefer host_alias over host. And - # prefer i686 (what external dev tools use) to i386 (what ruby's configure.ac emits). - host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] - host.gsub(/i386/, "i686") -end + lib_dirs = [ + "/usr/local/lib", + "/opt/homebrew/lib", + "/usr/lib" + ] -def target_arch - RbConfig::CONFIG['arch'] -end + dir_config("re2", header_dirs, lib_dirs) -def with_temp_dir - Dir.mktmpdir do |temp_dir| - Dir.chdir(temp_dir) do - yield + unless have_library("re2") + abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" + end end - end -end -# -# main -# -do_help if arg_config('--help') + def build_with_vendored_libraries + message "Building re2 using packaged libraries.\n" -if ENV["CC"] - RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] - RbConfig::CONFIG["CC"] = ENV["CC"] -end + abseil_recipe, re2_recipe = load_recipes -if ENV["CXX"] - RbConfig::MAKEFILE_CONFIG["CXX"] = ENV["CXX"] - RbConfig::CONFIG["CXX"] = ENV["CXX"] -end + process_recipe(abseil_recipe) do |recipe| + recipe.configure_options << '-DABSL_PROPAGATE_CXX_STD=ON' + # Workaround for https://github.com/abseil/abseil-cpp/issues/1510 + recipe.configure_options << '-DCMAKE_CXX_FLAGS=-DABSL_FORCE_WAITER_MODE=4' if MiniPortile.windows? + end -def build_extension(static_p = false) - # Enable optional warnings but disable deprecated register warning for Ruby 2.6 support - $CFLAGS << " -Wall -Wextra -funroll-loops" - $CPPFLAGS << " -Wno-register" + process_recipe(re2_recipe) do |recipe| + recipe.configure_options += [ + # Specify Abseil's path so RE2 will prefer that over any system Abseil + "-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", + '-DCMAKE_CXX_FLAGS=-DNDEBUG' + ] + end - # Pass -x c++ to force gcc to compile the test program - # as C++ (as it will end in .c by default). - compile_options = +"-x c++" + pc_file = File.join(re2_recipe.lib_path, 'pkgconfig', 're2.pc') + pkg_config_paths = [ + File.join(abseil_recipe.lib_path, 'pkgconfig'), + File.join(re2_recipe.lib_path, 'pkgconfig') + ] - have_library("stdc++") - have_header("stdint.h") - have_func("rb_gc_mark_movable") # introduced in Ruby 2.7 + static_pkg_config(pc_file, pkg_config_paths) + end - if !static_p and !have_library("re2") - abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" - end + def build_extension + # Enable optional warnings but disable deprecated register warning for Ruby 2.6 support + $CFLAGS << " -Wall -Wextra -funroll-loops" + $CXXFLAGS << " -Wall -Wextra -funroll-loops" + $CPPFLAGS << " -Wno-register" - minimal_program = < -int main() { return 0; } -SRC + # Pass -x c++ to force gcc to compile the test program + # as C++ (as it will end in .c by default). + compile_options = +"-x c++" - re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do - !try_compile(minimal_program, compile_options) - end + have_library("stdc++") + have_header("stdint.h") + have_func("rb_gc_mark_movable") # introduced in Ruby 2.7 + + minimal_program = <<~SRC + #include + int main() { return 0; } + SRC - if re2_requires_version_flag - # Recent versions of re2 depend directly on abseil, which requires a - # compiler with C++14 support (see - # https://github.com/abseil/abseil-cpp/issues/1127 and - # https://github.com/abseil/abseil-cpp/issues/1431). However, the - # `std=c++14` flag doesn't appear to suffice; we need at least - # `std=c++17`. - abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| - checking_for("re2 that compiles with #{std} standard") do - if try_compile(minimal_program, compile_options + " -std=#{std}") - compile_options << " -std=#{std}" - $CPPFLAGS << " -std=#{std}" - - true + re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do + !try_compile(minimal_program, compile_options) + end + + if re2_requires_version_flag + # Recent versions of re2 depend directly on abseil, which requires a + # compiler with C++14 support (see + # https://github.com/abseil/abseil-cpp/issues/1127 and + # https://github.com/abseil/abseil-cpp/issues/1431). However, the + # `std=c++14` flag doesn't appear to suffice; we need at least + # `std=c++17`. + abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| + checking_for("re2 that compiles with #{std} standard") do + if try_compile(minimal_program, compile_options + " -std=#{std}") + compile_options << " -std=#{std}" + $CPPFLAGS << " -std=#{std}" + + true + end + end end end - end - end - # Determine which version of re2 the user has installed. - # Revision d9f8806c004d added an `endpos` argument to the - # generic Match() function. - # - # To test for this, try to compile a simple program that uses - # the newer form of Match() and set a flag if it is successful. - checking_for("RE2::Match() with endpos argument") do - test_re2_match_signature = < - -int main() { - RE2 pattern("test"); - re2::StringPiece *match; - pattern.Match("test", 0, 0, RE2::UNANCHORED, match, 0); - - return 0; -} -SRC - - if try_compile(test_re2_match_signature, compile_options) - $defs.push("-DHAVE_ENDPOS_ARGUMENT") - end - end + # Determine which version of re2 the user has installed. + # Revision d9f8806c004d added an `endpos` argument to the + # generic Match() function. + # + # To test for this, try to compile a simple program that uses + # the newer form of Match() and set a flag if it is successful. + checking_for("RE2::Match() with endpos argument") do + test_re2_match_signature = <<~SRC + #include + + int main() { + RE2 pattern("test"); + re2::StringPiece *match; + pattern.Match("test", 0, 0, RE2::UNANCHORED, match, 0); + + return 0; + } + SRC + + if try_compile(test_re2_match_signature, compile_options) + $defs.push("-DHAVE_ENDPOS_ARGUMENT") + end + end - checking_for("RE2::Set::Match() with error information") do - test_re2_set_match_signature = < -#include -#include + checking_for("RE2::Set::Match() with error information") do + test_re2_set_match_signature = <<~SRC + #include + #include + #include -int main() { - RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); - s.Add("foo", NULL); - s.Compile(); + int main() { + RE2::Set s(RE2::DefaultOptions, RE2::UNANCHORED); + s.Add("foo", NULL); + s.Compile(); - std::vector v; - RE2::Set::ErrorInfo ei; - s.Match("foo", &v, &ei); + std::vector v; + RE2::Set::ErrorInfo ei; + s.Match("foo", &v, &ei); - return 0; -} -SRC + return 0; + } + SRC - if try_compile(test_re2_set_match_signature, compile_options) - $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + if try_compile(test_re2_set_match_signature, compile_options) + $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + end + end end - end -end -def process_recipe(recipe) - cross_build_p = config_cross_build? - message "Cross build is #{cross_build_p ? "enabled" : "disabled"}.\n" + def static_pkg_config(pc_file, pkg_config_paths) + ENV["PKG_CONFIG_PATH"] = [*pkg_config_paths, ENV["PKG_CONFIG_PATH"]].compact.join(File::PATH_SEPARATOR) - recipe.host = target_host - # Ensure x64-mingw-ucrt and x64-mingw32 use different library paths since the host - # is the same (x86_64-w64-mingw32). - recipe.target = File.join(recipe.target, target_arch) if cross_build_p + static_library_paths = minimal_pkg_config(pc_file, '--libs-only-L', '--static') + .shellsplit + .map { |flag| flag.delete_prefix('-L') } - yield recipe + $LIBPATH = static_library_paths | $LIBPATH - checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" - name = recipe.name - version = recipe.version + # Replace all -l flags that can be found in one of the static library + # paths with the absolute path instead. + minimal_pkg_config(pc_file, '--libs-only-l', '--static') + .shellsplit + .each do |flag| + lib = "lib#{flag.delete_prefix('-l')}.#{$LIBEXT}" - if File.exist?(checkpoint) - message("Building re2 with a packaged version of #{name}-#{version}.\n") - else - message(<<~EOM) - ---------- IMPORTANT NOTICE ---------- - Building re2 with a packaged version of #{name}-#{version}. - Configuration options: #{recipe.configure_options.shelljoin} - EOM + if (static_lib_path = static_library_paths.find { |path| File.exist?(File.join(path, lib)) }) + $libs << ' ' << File.join(static_lib_path, lib).shellescape + else + $libs << ' ' << flag.shellescape + end + end - unless recipe.patch_files.empty? - message("The following patches are being applied:\n") + append_ldflags(minimal_pkg_config(pc_file, '--libs-only-other', '--static')) - recipe.patch_files.each do |patch| - message(" - %s\n" % File.basename(patch)) - end + incflags = minimal_pkg_config(pc_file, '--cflags-only-I') + $INCFLAGS = [incflags, $INCFLAGS].join(" ").strip + + cflags = minimal_pkg_config(pc_file, '--cflags-only-other') + $CFLAGS = [$CFLAGS, cflags].join(" ").strip + $CXXFLAGS = [$CXXFLAGS, cflags].join(" ").strip end - # Use a temporary base directory to reduce filename lengths since - # Windows can hit a limit of 250 characters (CMAKE_OBJECT_PATH_MAX). - with_temp_dir { recipe.cook } + def process_recipe(recipe) + cross_build_p = config_cross_build? + message "Cross build is #{cross_build_p ? "enabled" : "disabled"}.\n" - FileUtils.touch(checkpoint) - end + recipe.host = target_host + # Ensure x64-mingw-ucrt and x64-mingw32 use different library paths since the host + # is the same (x86_64-w64-mingw32). + recipe.target = File.join(recipe.target, target_arch) if cross_build_p - recipe.activate -end + yield recipe -def build_with_system_libraries - header_dirs = [ - "/usr/local/include", - "/opt/homebrew/include", - "/usr/include" - ] + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + name = recipe.name + version = recipe.version - lib_dirs = [ - "/usr/local/lib", - "/opt/homebrew/lib", - "/usr/lib" - ] + if File.exist?(checkpoint) + message("Building re2 with a packaged version of #{name}-#{version}.\n") + else + message(<<~EOM) + ---------- IMPORTANT NOTICE ---------- + Building re2 with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM - dir_config("re2", header_dirs, lib_dirs) + # Use a temporary base directory to reduce filename lengths since + # Windows can hit a limit of 250 characters (CMAKE_OBJECT_PATH_MAX). + Dir.mktmpdir { |dir| Dir.chdir(dir) { recipe.cook } } - build_extension -end - -def libflag_to_filename(ldflag) - case ldflag - when /\A-l(.+)/ - "lib#{Regexp.last_match(1)}.#{$LIBEXT}" - end -end + FileUtils.touch(checkpoint) + end + end -# This method does a number of things to ensure the final shared library -# is compiled statically with the vendored libraries: -# -# 1. For -L flags, ensure that any `ports` paths are prioritized just -# in case there are installed libraries that might take precedence. -# 2. For -l flags, convert the library to the static library with a -# full path and substitute the absolute static library. For example, -# -lre2 maps to /path/to/ports//libre2//lib/libre2.a. -# -# This is needed because when building the extension, Ruby appears to -# insert `-L#{RbConfig::CONFIG['exec_prefix']}/lib` first. If libre2 is -# in installed in that location then the extension will link against the -# system library instead of the vendored library. -def add_flag(arg, lib_paths) - case arg - when /\A-L(.+)\z/ - # Prioritize ports' directories - lib_dir = Regexp.last_match(1) - $LIBPATH = - if lib_dir.start_with?(PACKAGE_ROOT_DIR + "/") - [lib_dir] | $LIBPATH + # See MiniPortile2's minimal_pkg_config: + # https://github.com/flavorjones/mini_portile/blob/52fb0bc41c89a10f1ac7b5abcf0157e059194374/lib/mini_portile2/mini_portile.rb#L760-L783 + # and Ruby's pkg_config: + # https://github.com/ruby/ruby/blob/c505bb0ca0fd61c7ae931d26451f11122a2644e9/lib/mkmf.rb#L1916-L2004 + def minimal_pkg_config(pc_file, *options) + if ($PKGCONFIG ||= + (pkgconfig = MakeMakefile.with_config("pkg-config") {MakeMakefile.config_string("PKG_CONFIG") || "pkg-config"}) && + MakeMakefile.find_executable0(pkgconfig) && pkgconfig) + pkgconfig = $PKGCONFIG else - $LIBPATH | [lib_dir] + raise RuntimeError, "pkg-config is not found" end - when /\A-l./ - filename = libflag_to_filename(arg) - added = false - lib_paths.each do |path| - static_lib = File.join(path, filename) + response = xpopen([pkgconfig, *options, pc_file], err: %i[child out], &:read) + raise RuntimeError, response unless $?.success? - next unless File.exist?(static_lib) - - $LDFLAGS << " " << static_lib - added = true - break + response.strip end - append_ldflags(arg.shellescape) unless added - else - append_ldflags(arg.shellescape) - end -end - -def add_static_ldflags(flags, lib_paths) - flags.strip.shellsplit.each { |flag| add_flag(flag, lib_paths) } -end - -def build_with_vendored_libraries - message "Building re2 using packaged libraries.\n" + def config_system_libraries? + enable_config("system-libraries", ENV.key?('RE2_USE_SYSTEM_LIBRARIES')) + end - abseil_recipe, re2_recipe = load_recipes + def config_cross_build? + enable_config("cross-build") + end - process_recipe(abseil_recipe) do |recipe| - recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON', '-DCMAKE_CXX_VISIBILITY_PRESET=hidden'] - # Workaround for https://github.com/abseil/abseil-cpp/issues/1510 - recipe.configure_options += ['-DCMAKE_CXX_FLAGS=-DABSL_FORCE_WAITER_MODE=4'] if windows? - end + # We use 'host' to set compiler prefix for cross-compiling. Prefer host_alias over host. And + # prefer i686 (what external dev tools use) to i386 (what ruby's configure.ac emits). + def target_host + host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] + host.gsub(/i386/, "i686") + end - process_recipe(re2_recipe) do |recipe| - recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG', - '-DCMAKE_CXX_VISIBILITY_PRESET=hidden'] + def target_arch + RbConfig::CONFIG['arch'] + end end - - dir_config("re2", File.join(re2_recipe.path, 'include'), File.join(re2_recipe.path, 'lib')) - dir_config("abseil", File.join(abseil_recipe.path, 'include'), File.join(abseil_recipe.path, 'lib')) - - pkg_config_paths = [ - "#{abseil_recipe.path}/lib/pkgconfig", - "#{re2_recipe.path}/lib/pkgconfig" - ].join(File::PATH_SEPARATOR) - - pkg_config_paths = "#{ENV['PKG_CONFIG_PATH']}#{File::PATH_SEPARATOR}#{pkg_config_paths}" if ENV['PKG_CONFIG_PATH'] - - ENV['PKG_CONFIG_PATH'] = pkg_config_paths - pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc') - - raise 'Please install the `pkg-config` utility!' unless find_executable('pkg-config') - - # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. - flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) - - raise 'Unable to run pkg-config --libs --static' unless $?.success? - - lib_paths = [File.join(abseil_recipe.path, 'lib'), File.join(re2_recipe.path, 'lib')] - add_static_ldflags(flags, lib_paths) - build_extension(true) end -if config_system_libraries? - build_with_system_libraries -else - build_with_vendored_libraries +extconf = RE2::Extconf.new + +if arg_config('--help') + extconf.print_help + exit!(true) end -create_makefile("re2") +extconf.configure diff --git a/ext/re2/recipes.rb b/ext/re2/recipes.rb index 1b40f98..b856c1b 100644 --- a/ext/re2/recipes.rb +++ b/ext/re2/recipes.rb @@ -11,26 +11,6 @@ PACKAGE_ROOT_DIR = File.expand_path('../..', __dir__) REQUIRED_MINI_PORTILE_VERSION = '~> 2.8.5' # keep this version in sync with the one in the gemspec -def build_recipe(name, version) - require 'rubygems' - gem('mini_portile2', REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time - require 'mini_portile2' - - MiniPortileCMake.new(name, version).tap do |recipe| - recipe.target = File.join(PACKAGE_ROOT_DIR, 'ports') - recipe.configure_options += [ - # abseil needs a C++14 compiler - '-DCMAKE_CXX_STANDARD=14', - # needed for building the C extension shared library with -fPIC - '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', - # ensures pkg-config and installed libraries will be in lib, not lib64 - '-DCMAKE_INSTALL_LIBDIR=lib' - ] - - yield recipe - end -end - def load_recipes require 'yaml' dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml')) @@ -51,3 +31,24 @@ def load_recipes [abseil_recipe, re2_recipe] end + +def build_recipe(name, version) + require 'rubygems' + gem('mini_portile2', REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time + require 'mini_portile2' + + MiniPortileCMake.new(name, version).tap do |recipe| + recipe.target = File.join(PACKAGE_ROOT_DIR, 'ports') + recipe.configure_options += [ + # abseil needs a C++14 compiler + '-DCMAKE_CXX_STANDARD=14', + # needed for building the C extension shared library with -fPIC + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', + # ensures pkg-config and installed libraries will be in lib, not lib64 + '-DCMAKE_INSTALL_LIBDIR=lib', + '-DCMAKE_CXX_VISIBILITY_PRESET=hidden' + ] + + yield recipe + end +end