[GR-57603] Import Ruby 3.3.5

PullRequest: truffleruby/4366

.gitignore

@@ -43,8 +43,10 @@ src/main/c/**/*.dylib
 src/main/c/**/extconf.h
 src/main/c/**/mkmf.log
 src/main/c/*/Makefile
+!src/main/c/prism-gem/Makefile
 !src/main/c/spawn-helper/Makefile
 !src/main/c/truffleposix/Makefile
+!src/main/c/yarp/Makefile
 src/main/c/cext-trampoline/trampoline.c
 src/main/c/cext/wrappers.c
 src/main/c/etc/constdefs.h

.ruby-version

@@ -1 +1 @@
-3.2.4
+3.3.5

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 New features:
 
+* Updated to Ruby 3.3.5 (#3681, @andrykonchin, @eregon).
+
 Bug fixes:
 
 * Fix `Module#name` called inside the `Module#const_added` callback when the module is defined in the top-level scope (#3683, @andrykonchin).
@@ -20,6 +22,7 @@ Compatibility:
 * Support `Time.new` with String argument and error when invalid (#3693, @rwstauner).
 * Implement `rb_enc_interned_str()` (#3703, @Th3-M4jor).
 * Implement `rb_hash_bulk_insert()` (#3705, @Th3-M4jor).
+* Remove deprecated `Pathname#{taint,untaint}` methods (#3681, @andrykonchin).
 
 Performance:
 

ci.jsonnet

@@ -393,6 +393,20 @@ local part_definitions = {
       is_after+:: ["$.run.generate_native_config"],
       run+: jt(["check_native_configuration"]) + jt(["check_config_header"]),
     },
+
+    retag_mri_tests: {
+      run+: jt(["retag"]) + [
+        ["set-export", "DATE", ["date", "+%F"]],
+        ["set-export", "TRUFFLERUBY_BRANCH", "retag-mri-tests-$DATE"],
+        ["git", "checkout", "-b", "$TRUFFLERUBY_BRANCH"],
+        ["git", "config", "--local", "user.email", "[email protected]"],
+        ["git", "config", "--local", "user.name", "ol-automation_ww"],
+        ["git", "add", "."],
+        ["git", "commit", "-m", "Retag MRI tests"],
+        ["git", "remote", "-v"],
+        ["git", "push", "-f", "origin", "HEAD"],
+      ],
+    },
   },
 
   benchmark: {
@@ -513,8 +527,8 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       # Order: platform, jdk, mx_env. Keep aligned for an easy visual comparison.
       "ruby-test-specs-linux-amd64-stable":    $.platform.linux  + $.jdk.stable + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:20:00" },
       "ruby-test-specs-linux-amd64-latest":    $.platform.linux  + $.jdk.latest + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:20:00" },
-      "ruby-test-specs-darwin-amd64-stable":   $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:40:00" },
-      "ruby-test-specs-darwin-amd64-latest":   $.platform.darwin_amd64 + $.jdk.latest + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:40:00" },
+      "ruby-test-specs-darwin-amd64-stable":   $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "02:00:00" },
+      "ruby-test-specs-darwin-amd64-latest":   $.platform.darwin_amd64 + $.jdk.latest + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "02:00:00" },
       "ruby-test-specs-darwin-aarch64-stable": $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:40:00" },
       "ruby-test-specs-darwin-aarch64-latest": $.platform.darwin_aarch64 + $.jdk.latest + $.env.jvm + gate_no_build + $.use.build + $.run.test_unit_tck + native_config + $.run.test_specs + { timelimit: "01:40:00" },
       "ruby-test-fast-linux-aarch64":   $.platform.linux_aarch64 + $.jdk.latest + $.env.jvm + gate + $.run.test_fast + native_config + { timelimit: "45:00" },
@@ -698,6 +712,8 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
     "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.latest + shared + native_config,
     "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.latest + shared + native_config,
     "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.latest + shared + native_config,
+
+    "ruby-retag-mri-tests-linux-amd64":           $.platform.linux + $.jdk.latest + shared + $.env.native + $.use.build + $.run.retag_mri_tests + { timelimit: "02:00:00" },
   },
 
   builds:

doc/contributor/how-to-guide.md

@@ -1627,7 +1627,7 @@ The `jt` CLI tool has a lot of commands but there are very few basic ones that y
 - `jt sync` - to synchronize changes in Standard Library Ruby source code and apply them without rebuilding TruffleRuby
 - `jt tag` - ruby/spec only
 - `jt untag <path-to-file>` - (ruby/spec only) regenerate an exclude file for particular ruby/spec file (and list all the failed test cases)
-- `jt retag <path-to-file>` - (MRI test only) regenerate an exclude file for particular MRI test file (and list all the failed test cases)
+- `jt retag <path-to-file>` - (MRI test only) regenerate an exclude file for particular MRI test file (and list all the failed test cases). Do nothing if there aren't any already excluded cases for this file.
 - `jt lint fast` - to run some linter checks (that are run on CI) locally
 - `jt rubocop` - to run Ruby source code linter separately from other linter checks.
     - `jt rubocop -a` - to correct safely Rubocop warnings automatically

doc/contributor/updating-ruby.md

@@ -135,12 +135,19 @@ rm -rf lib/gems/specifications
 cd ~/.rubies/ruby-$VERSION
 cp -R lib/ruby/gems/*.0/gems $TRUFFLERUBY/lib/gems
 cp -R lib/ruby/gems/*.0/specifications $TRUFFLERUBY/lib/gems
+rm -f lib/gems/gems/**/*.{o,a,so,bundle} lib/gems/gems/**/{Makefile,extconf.h,mkmf.log} lib/gems/gems/**/*.mk
 
 cd $TRUFFLERUBY
 rm -rf lib/gems/gems/typeprof-* lib/gems/specifications/typeprof-*.gemspec
 ruby tool/patch-default-gemspecs.rb
 ```
 
+Then review bundled gems with extensions like `lib/gems/gems/debug-*` and `lib/gems/gems/rbs-*`
+to ensure no build artifacts/generated files are committed, only "sources".
+
+Update the `ruby/prism` default gem with `tool/import-prism.sh` script. See the "Update Prism" section
+in the [Prism](prism.md) document.
+
 ## Updating exe/ executables
 
 ```
@@ -162,7 +169,7 @@ Update all of these:
   * with bundled gem versions provided by `cat ../ruby/gems/bundled_gems | sort`,
   * default gem versions provided by `ls -l lib/gems/specifications/default`
   * and `gem` gem version provided by `grep 'VERSION =' lib/mri/rubygems.rb`
-* Also update version numbers for `debug` and `rbs` in `src/main/c/Makefile`, `mx.truffleruby/suite.py` and `lib/gems/gems/debug-*/ext/debug/extconf.rb`.
+* Also update version numbers for `debug`, `racc`, and `rbs` in `src/main/c/Makefile`, `mx.truffleruby/suite.py` and `lib/gems/gems/debug-*/ext/debug/extconf.rb`.
 * Copy and paste `-h` and `--help` output to `RubyLauncher` (instructions are in the end of the file `src/launcher/java/org/truffleruby/launcher/RubyLauncher.java`)
 * This is a good time to get `jt build` working.
 * Copy and paste the TruffleRuby `--help` output to `doc/user/options.md` (e.g., with `jt ruby --help | xsel -b`)

doc/legal/bundled_gems

@@ -1,16 +1,23 @@
-# gem-name version-to-bundle repository-url [optional-commit-hash-to-test-or-defaults-to-v-version]
-minitest        5.16.3  https://github.com/seattlerb/minitest
+# gem-name version repository-url [revision]
+#
+# - gem-name: gem name to bundle
+# - version: released version to bundle
+# - repository-url: URL from where clone for test
+# - revision: revision in repository-url to test
+#   if `revision` is not given, "v"+`version` or `version` will be used.
+minitest        5.20.0  https://github.com/minitest/minitest
 power_assert    2.0.3   https://github.com/ruby/power_assert
-rake            13.0.6  https://github.com/ruby/rake
-test-unit       3.5.7   https://github.com/test-unit/test-unit
-rexml           3.2.5   https://github.com/ruby/rexml
-rss             0.2.9   https://github.com/ruby/rss
-net-ftp         0.2.1   https://github.com/ruby/net-ftp
-net-imap        0.3.4.1   https://github.com/ruby/net-imap
+rake            13.1.0  https://github.com/ruby/rake
+test-unit       3.6.1   https://github.com/test-unit/test-unit
+rexml           3.3.6   https://github.com/ruby/rexml
+rss             0.3.1   https://github.com/ruby/rss
+net-ftp         0.3.4   https://github.com/ruby/net-ftp
+net-imap        0.4.9.1   https://github.com/ruby/net-imap
 net-pop         0.1.2   https://github.com/ruby/net-pop
-net-smtp        0.3.4   https://github.com/ruby/net-smtp
+net-smtp        0.4.0.1   https://github.com/ruby/net-smtp
 matrix          0.4.2   https://github.com/ruby/matrix
 prime           0.1.2   https://github.com/ruby/prime
-rbs             2.8.2   https://github.com/ruby/rbs
-typeprof        0.21.3  https://github.com/ruby/typeprof
-debug           1.7.1   https://github.com/ruby/debug
+rbs             3.4.0   https://github.com/ruby/rbs
+typeprof        0.21.9  https://github.com/ruby/typeprof
+debug           1.9.1   https://github.com/ruby/debug
+racc            1.7.3   https://github.com/ruby/racc

doc/legal/legal.md

@@ -18,7 +18,7 @@ See `epl-2.0.txt`, `gpl-2.txt`, `lgpl-2.1.txt`.
 ## MRI
 
 The standard implementation of Ruby is MRI. TruffleRuby contains code from MRI
-version 3.2.4, including:
+version 3.3.5, including:
 
 * the standard library in `lib/mri`, 
 * Ruby C extension API in `lib/cext/include` and `src/main/c/cext`, 
@@ -124,6 +124,10 @@ power_assert is copyright Kazuki Tsujimoto and is available under the same licen
 
 prime is under the same copyright and licence as MRI (see `ruby-bsdl.txt`).
 
+#### racc
+
+racc is under the same copyright and licence as MRI (see `ruby-bsdl.txt`).
+
 #### rake
 
 Rake is copyright Jim Weirich and is available under an MIT licence (see `mit.txt`).

doc/user/compatibility.md

@@ -7,7 +7,7 @@ permalink: /reference-manual/ruby/Compatibility/
 # Compatibility
 
 TruffleRuby aims to be fully compatible with the standard implementation of
-Ruby, MRI, version 3.2.4, [including C extensions](#c-extension-compatibility).
+Ruby, MRI, version 3.3.5, [including C extensions](#c-extension-compatibility).
 TruffleRuby is still in development, so it is not 100% compatible yet.
 
 TruffleRuby can run Rails and is compatible with many gems, including C extensions.

doc/user/options.md

@@ -11,6 +11,7 @@ TruffleRuby has the same command-line interface as our compatible MRI version.
 ```shell
 Usage: truffleruby [switches] [--] [programfile] [arguments]
   -0[octal]       specify record separator (\0, if no argument)
+                  (-00 for paragraph mode, -0777 for slurp mode)
   -a              autosplit mode with -n or -p (splits $_ into $F)
   -c              check syntax only
   -Cdirectory     cd to directory before executing your script
@@ -33,9 +34,11 @@ Usage: truffleruby [switches] [--] [programfile] [arguments]
                   set warning level; 0=silence, 1=medium, 2=verbose
   -x[directory]   strip off text before #!ruby line and perhaps cd to directory
   --copyright     print the copyright
-  --enable={jit|rubyopt|...}[,...], --disable={jit|rubyopt|...}[,...]
+  --enable={rubyopt|...}[,...]
+  --disable={rubyopt|...}[,...]
                   enable or disable features. see below for available features
-  --external-encoding=encoding, --internal-encoding=encoding
+  --external-encoding=encoding
+  --internal-encoding=encoding
                   specify the default external or internal character encoding
   --backtrace-limit=num
                   limit the maximum length of backtrace
@@ -55,6 +58,7 @@ Features:
 Warning categories:
   deprecated      deprecated features
   experimental    experimental features
+  performance     performance issues
 
 Runtime options:
   --polyglot                                   Run with all other guest languages accessible.

exe/gem

@@ -18,6 +18,8 @@ done
 exec "$(dirname $SELF_PATH)/ruby" "$SELF_PATH" "$@"
 
 #!ruby
+# frozen_string_literal: true
+
 #--
 # Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others.
 # All rights reserved.

exe/syntax_suggest

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# This file was generated by RubyGems.
+# The above lines match the format expected by rubygems/installer.rb check_executable_overwrite
+# bash section ignored by the Ruby interpreter
+
+# get the absolute path of the executable and resolve symlinks
+SELF_PATH=$(cd "$(dirname "$0")" && pwd -P)/$(basename "$0")
+while [ -h "$SELF_PATH" ]; do
+  # 1) cd to directory of the symlink
+  # 2) cd to the directory of where the symlink points
+  # 3) get the pwd
+  # 4) append the basename
+  DIR=$(dirname "$SELF_PATH")
+  SYM=$(readlink "$SELF_PATH")
+  SELF_PATH=$(cd "$DIR" && cd "$(dirname "$SYM")" && pwd)/$(basename "$SYM")
+done
+exec "$(dirname $SELF_PATH)/ruby" "$SELF_PATH" "$@"
+
+#!ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'syntax_suggest' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+Gem.use_gemdeps
+
+version = ">= 0.a"
+
+str = ARGV.first
+if str
+  str = str.b[/\A_(.*)_\z/, 1]
+  if str and Gem::Version.correct?(str)
+    version = str
+    ARGV.shift
+  end
+end
+
+if Gem.respond_to?(:activate_bin_path)
+load Gem.activate_bin_path('syntax_suggest', 'syntax_suggest', version)
+else
+gem "syntax_suggest", version
+load Gem.bin_path("syntax_suggest", "syntax_suggest", version)
+end

lib/cext/ABI_check.txt

@@ -1 +1 @@
-3
+2

lib/cext/include/ccan/list/list.h

@@ -635,14 +635,16 @@ static inline void ccan_list_prepend_list_(struct ccan_list_head *to,
 
 /* internal macros, do not use directly */
 #define ccan_list_for_each_off_dir_(h, i, off, dir)				\
-	for (i = ccan_list_node_to_off_(ccan_list_debug(h, CCAN_LIST_LOC)->n.dir,	\
+	for (i = 0, \
+	     i = ccan_list_node_to_off_(ccan_list_debug(h, CCAN_LIST_LOC)->n.dir, \
 				   (off));				\
 	ccan_list_node_from_off_((void *)i, (off)) != &(h)->n;		\
 	i = ccan_list_node_to_off_(ccan_list_node_from_off_((void *)i, (off))->dir, \
 			      (off)))
 
 #define ccan_list_for_each_safe_off_dir_(h, i, nxt, off, dir)		\
-	for (i = ccan_list_node_to_off_(ccan_list_debug(h, CCAN_LIST_LOC)->n.dir,	\
+	for (i = 0, \
+	     i = ccan_list_node_to_off_(ccan_list_debug(h, CCAN_LIST_LOC)->n.dir, \
 				   (off)),				\
 	nxt = ccan_list_node_to_off_(ccan_list_node_from_off_(i, (off))->dir,	\
 				(off));					\

lib/cext/include/internal.h

@@ -19,6 +19,8 @@
 #define LIKELY(x) RB_LIKELY(x)
 #define UNLIKELY(x) RB_UNLIKELY(x)
 
+#define numberof(array) ((int)(sizeof(array) / sizeof((array)[0])))
+
 #include "ruby/ruby.h"
 
 #define RBOOL(v) ((v) ? Qtrue : Qfalse)

lib/cext/include/internal/bignum.h

@@ -167,11 +167,9 @@ VALUE rb_str2big_gmp(VALUE arg, int base, int badcheck);
 VALUE rb_int_parse_cstr(const char *str, ssize_t len, char **endp, size_t *ndigits, int base, int flags);
 RUBY_SYMBOL_EXPORT_END
 
-MJIT_SYMBOL_EXPORT_BEGIN
 #if defined(HAVE_INT128_T)
 VALUE rb_int128t2big(int128_t n);
 #endif
-MJIT_SYMBOL_EXPORT_END
 
 /* sign: positive:1, negative:0 */
 static inline bool

lib/cext/include/internal/bits.h

@@ -34,8 +34,7 @@
 # include <stdlib.h>            /* for _byteswap_uint64 */
 #endif
 
-#if defined(HAVE_X86INTRIN_H) && ! defined(MJIT_HEADER)
-# /* Rule out MJIT_HEADER, which does not interface well with <immintrin.h> */
+#if defined(HAVE_X86INTRIN_H)
 # include <x86intrin.h>         /* for _lzcnt_u64 */
 #elif MSC_VERSION_SINCE(1310)
 # include <intrin.h>            /* for the following intrinsics */
@@ -119,12 +118,16 @@
     MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
 #endif
 
-#ifdef MUL_OVERFLOW_P
+#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG)
 # define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
+#else
+# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
+#endif
+
+#ifdef MUL_OVERFLOW_P
 # define MUL_OVERFLOW_LONG_P(a, b)      MUL_OVERFLOW_P(a, b)
 # define MUL_OVERFLOW_INT_P(a, b)       MUL_OVERFLOW_P(a, b)
 #else
-# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
 # define MUL_OVERFLOW_LONG_P(a, b)      MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
 # define MUL_OVERFLOW_INT_P(a, b)       MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
 #endif
@@ -235,7 +238,7 @@ nlz_int32(uint32_t x)
      * safety. */
     return (unsigned int)__lzcnt(x);
 
-#elif defined(__x86_64__) && defined(__LZCNT__) && ! defined(MJIT_HEADER)
+#elif defined(__x86_64__) && defined(__LZCNT__)
     return (unsigned int)_lzcnt_u32(x);
 
 #elif MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */
@@ -264,7 +267,7 @@ nlz_int64(uint64_t x)
 #if defined(_MSC_VER) && defined(__AVX2__)
     return (unsigned int)__lzcnt64(x);
 
-#elif defined(__x86_64__) && defined(__LZCNT__) && ! defined(MJIT_HEADER)
+#elif defined(__x86_64__) && defined(__LZCNT__)
     return (unsigned int)_lzcnt_u64(x);
 
 #elif defined(_WIN64) && MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */
@@ -395,9 +398,9 @@ rb_popcount32(uint32_t x)
 #else
     x = (x & 0x55555555) + (x >> 1 & 0x55555555);
     x = (x & 0x33333333) + (x >> 2 & 0x33333333);
-    x = (x & 0x0f0f0f0f) + (x >> 4 & 0x0f0f0f0f);
-    x = (x & 0x001f001f) + (x >> 8 & 0x001f001f);
-    x = (x & 0x0000003f) + (x >>16 & 0x0000003f);
+    x = (x & 0x07070707) + (x >> 4 & 0x07070707);
+    x = (x & 0x000f000f) + (x >> 8 & 0x000f000f);
+    x = (x & 0x0000001f) + (x >>16 & 0x0000001f);
     return (unsigned int)x;
 
 #endif
@@ -425,9 +428,9 @@ rb_popcount64(uint64_t x)
     x = (x & 0x5555555555555555) + (x >> 1 & 0x5555555555555555);
     x = (x & 0x3333333333333333) + (x >> 2 & 0x3333333333333333);
     x = (x & 0x0707070707070707) + (x >> 4 & 0x0707070707070707);
-    x = (x & 0x001f001f001f001f) + (x >> 8 & 0x001f001f001f001f);
-    x = (x & 0x0000003f0000003f) + (x >>16 & 0x0000003f0000003f);
-    x = (x & 0x000000000000007f) + (x >>32 & 0x000000000000007f);
+    x = (x & 0x000f000f000f000f) + (x >> 8 & 0x000f000f000f000f);
+    x = (x & 0x0000001f0000001f) + (x >>16 & 0x0000001f0000001f);
+    x = (x & 0x000000000000003f) + (x >>32 & 0x000000000000003f);
     return (unsigned int)x;
 
 #endif
@@ -450,7 +453,7 @@ rb_popcount_intptr(uintptr_t x)
 static inline int
 ntz_int32(uint32_t x)
 {
-#if defined(__x86_64__) && defined(__BMI__) && ! defined(MJIT_HEADER)
+#if defined(__x86_64__) && defined(__BMI__)
     return (unsigned)_tzcnt_u32(x);
 
 #elif MSC_VERSION_SINCE(1400)
@@ -472,7 +475,7 @@ ntz_int32(uint32_t x)
 static inline int
 ntz_int64(uint64_t x)
 {
-#if defined(__x86_64__) && defined(__BMI__) && ! defined(MJIT_HEADER)
+#if defined(__x86_64__) && defined(__BMI__)
     return (unsigned)_tzcnt_u64(x);
 
 #elif defined(_WIN64) && MSC_VERSION_SINCE(1400)