diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index 64c7082214e540..c80dec1b7eaa50 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -93,7 +93,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2 # v2.74.0 + - uses: taiki-e/install-action@cf39a74df4a72510be4e5b63348d61067f11e64a # v2.75.0 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index 06a3376e493c55..5c269774179c95 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -119,7 +119,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2 # v2.74.0 + - uses: taiki-e/install-action@cf39a74df4a72510be4e5b63348d61067f11e64a # v2.75.0 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/ext/openssl/ossl_kdf.c b/ext/openssl/ossl_kdf.c index 1f280164405aed..f70b7f6cf9a9f8 100644 --- a/ext/openssl/ossl_kdf.c +++ b/ext/openssl/ossl_kdf.c @@ -7,6 +7,27 @@ static VALUE mKDF, eKDF; +struct pbkdf2_hmac_args { + char *pass; + int passlen; + unsigned char *salt; + int saltlen; + int iters; + const EVP_MD *md; + int len; + unsigned char *out; +}; + +static void * +pbkdf2_hmac_nogvl(void *args_) +{ + struct pbkdf2_hmac_args *args = (struct pbkdf2_hmac_args *)args_; + int ret = PKCS5_PBKDF2_HMAC(args->pass, args->passlen, args->salt, + args->saltlen, args->iters, args->md, + args->len, args->out); + return (void *)(uintptr_t)ret; +} + /* * call-seq: * KDF.pbkdf2_hmac(pass, salt:, iterations:, length:, hash:) -> aString @@ -35,9 +56,9 @@ static VALUE mKDF, eKDF; static VALUE kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self) { - VALUE pass, salt, opts, kwargs[4], str, md_holder; + VALUE pass, salt, opts, kwargs[4], str, md_holder, pass_tmp, salt_tmp; static ID kwargs_ids[4]; - int iters, len; + int passlen, saltlen, iters, len; const EVP_MD *md; if (!kwargs_ids[0]) { @@ -54,18 +75,56 @@ kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self) iters = NUM2INT(kwargs[1]); len = NUM2INT(kwargs[2]); md = ossl_evp_md_fetch(kwargs[3], &md_holder); - - str = rb_str_new(0, len); - if (!PKCS5_PBKDF2_HMAC(RSTRING_PTR(pass), RSTRING_LENINT(pass), - (unsigned char *)RSTRING_PTR(salt), - RSTRING_LENINT(salt), iters, md, len, - (unsigned char *)RSTRING_PTR(str))) + passlen = RSTRING_LENINT(pass); + saltlen = RSTRING_LENINT(salt); + str = rb_str_new(NULL, len); + struct pbkdf2_hmac_args args = { + .pass = ALLOCV(pass_tmp, passlen), + .passlen = passlen, + .salt = ALLOCV(salt_tmp, saltlen), + .saltlen = saltlen, + .iters = iters, + .md = md, + .len = len, + .out = (unsigned char *)RSTRING_PTR(str), + }; + memcpy(args.pass, RSTRING_PTR(pass), passlen); + memcpy(args.salt, RSTRING_PTR(salt), saltlen); + if (!rb_thread_call_without_gvl(pbkdf2_hmac_nogvl, &args, NULL, NULL)) ossl_raise(eKDF, "PKCS5_PBKDF2_HMAC"); - + OPENSSL_cleanse(args.pass, passlen); + ALLOCV_END(pass_tmp); + ALLOCV_END(salt_tmp); return str; } #if defined(HAVE_EVP_PBE_SCRYPT) +struct scrypt_args { + char *pass; + size_t passlen; + unsigned char *salt; + size_t saltlen; + uint64_t N, r, p; + size_t len; + unsigned char *out; +}; + +static void * +scrypt_nogvl(void *args_) +{ + struct scrypt_args *args = (struct scrypt_args *)args_; + /* + * OpenSSL uses 32MB by default (if zero is specified), which is too + * small. Let's not limit memory consumption but just let malloc() fail + * inside OpenSSL. The amount is controllable by other parameters. + */ + uint64_t maxmem = UINT64_MAX; + int ret = EVP_PBE_scrypt(args->pass, args->passlen, + args->salt, args->saltlen, args->N, args->r, + args->p, maxmem, args->out, args->len); + return (void *)(uintptr_t)ret; +} + /* * call-seq: * KDF.scrypt(pass, salt:, N:, r:, p:, length:) -> aString @@ -101,10 +160,11 @@ kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self) static VALUE kdf_scrypt(int argc, VALUE *argv, VALUE self) { - VALUE pass, salt, opts, kwargs[5], str; + VALUE pass, salt, opts, kwargs[5], str, pass_tmp, salt_tmp; static ID kwargs_ids[5]; - size_t len; - uint64_t N, r, p, maxmem; + size_t passlen, saltlen; + long len; + uint64_t N, r, p; if (!kwargs_ids[0]) { kwargs_ids[0] = rb_intern_const("salt"); @@ -122,19 +182,27 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self) r = NUM2UINT64T(kwargs[2]); p = NUM2UINT64T(kwargs[3]); len = NUM2LONG(kwargs[4]); - /* - * OpenSSL uses 32MB by default (if zero is specified), which is too small. - * Let's not limit memory consumption but just let malloc() fail inside - * OpenSSL. The amount is controllable by other parameters. - */ - maxmem = SIZE_MAX; - - str = rb_str_new(0, len); - if (!EVP_PBE_scrypt(RSTRING_PTR(pass), RSTRING_LEN(pass), - (unsigned char *)RSTRING_PTR(salt), RSTRING_LEN(salt), - N, r, p, maxmem, (unsigned char *)RSTRING_PTR(str), len)) + passlen = RSTRING_LEN(pass); + saltlen = RSTRING_LEN(salt); + str = rb_str_new(NULL, len); + struct scrypt_args args = { + .pass = ALLOCV(pass_tmp, passlen), + .passlen = passlen, + .salt = ALLOCV(salt_tmp, saltlen), + .saltlen = saltlen, + .N = N, + .r = r, + .p = p, + .len = len, + .out = (unsigned char *)RSTRING_PTR(str), + }; + memcpy(args.pass, RSTRING_PTR(pass), passlen); + memcpy(args.salt, RSTRING_PTR(salt), saltlen); + if (!rb_thread_call_without_gvl(scrypt_nogvl, &args, NULL, NULL)) ossl_raise(eKDF, "EVP_PBE_scrypt"); - + OPENSSL_cleanse(args.pass, passlen); + ALLOCV_END(pass_tmp); + ALLOCV_END(salt_tmp); return str; } #endif diff --git a/file.c b/file.c index 6d715269f849cf..b4560bea630c03 100644 --- a/file.c +++ b/file.c @@ -5977,7 +5977,7 @@ rb_f_test(int argc, VALUE *argv, VALUE _) * File.atime(filepath) # => 2026-04-01 11:58:11.922614 -0500 * stat.atime # => 2026-04-01 11:51:38.0014518 -0500 * File.delete(filepath) - stat.atime # => 2026-04-01 11:51:38.0014518 -0500 + * stat.atime # => 2026-04-01 11:51:38.0014518 -0500 * * === OS-Dependencies * diff --git a/gc.c b/gc.c index 3f32f11eff376a..f05b1a691375ca 100644 --- a/gc.c +++ b/gc.c @@ -1313,18 +1313,7 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) switch (flags & RUBY_T_MASK) { case T_IMEMO: - switch (imemo_type(obj)) { - case imemo_constcache: - case imemo_cref: - case imemo_ifunc: - case imemo_memo: - case imemo_svar: - case imemo_callcache: - case imemo_throw_data: - return false; - default: - return true; - } + return rb_imemo_needs_cleanup_p(obj); case T_DATA: case T_OBJECT: diff --git a/internal/imemo.h b/internal/imemo.h index c5c0d005f1ffa8..4f2c4ebfbf6e98 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -40,7 +40,7 @@ enum imemo_type { imemo_callinfo = 10, imemo_callcache = 11, imemo_constcache = 12, - imemo_fields = 13, + imemo_fields = 13, }; /* CREF (Class REFerence) is defined in method.h */ @@ -292,4 +292,30 @@ rb_imemo_fields_complex_tbl(VALUE fields_obj) return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table; } +static inline bool +rb_imemo_needs_cleanup_p(VALUE obj) +{ + switch (imemo_type(obj)) { + case imemo_constcache: + case imemo_cref: + case imemo_ifunc: + case imemo_memo: + case imemo_svar: + case imemo_callcache: + case imemo_throw_data: + return false; + + case imemo_env: + case imemo_ment: + case imemo_iseq: + case imemo_callinfo: + case imemo_fields: + return true; + + case imemo_tmpbuf: + return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL; + } + UNREACHABLE_RETURN(true); +} + #endif /* INTERNAL_IMEMO_H */ diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index ed66ba9a48175c..7a616a4bb37bbc 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -71,7 +71,7 @@ Any periods in the configuration keys must be replaced with two underscores when The following is a list of all configuration keys and their purpose\. You can learn more about their operation in bundle install(1) \fIbundle\-install\.1\.html\fR\. .TP \fBapi_request_size\fR (\fBBUNDLE_API_REQUEST_SIZE\fR) -Configure how many dependencies to fetch when resolving the specifications\. This configuration is only used when fetchig specifications from RubyGems servers that didn't implement the Compact Index API\. Defaults to 100\. +Configure how many dependencies to fetch when resolving the specifications\. This configuration is only used when fetching specifications from RubyGems servers that didn't implement the Compact Index API\. Defaults to 100\. .TP \fBauto_install\fR (\fBBUNDLE_AUTO_INSTALL\fR) Automatically run \fBbundle install\fR when gems are missing\. diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index b70293cfeddadc..8a6adb99106e75 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -108,7 +108,7 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `api_request_size` (`BUNDLE_API_REQUEST_SIZE`): Configure how many dependencies to fetch when resolving the specifications. - This configuration is only used when fetchig specifications from RubyGems + This configuration is only used when fetching specifications from RubyGems servers that didn't implement the Compact Index API. Defaults to 100. * `auto_install` (`BUNDLE_AUTO_INSTALL`): diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 877343b5fb4d3f..cee5196f3082c9 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -511,11 +511,11 @@ def extension_cache_slug(spec) remote.cache_slug end - # We are using a mutex to reaed and write from/to the hash. + # We are using a mutex to read and write from/to the hash. # The reason this double synchronization was added is for performance - # and lock the mutex for the shortest possible amount of time. Otherwise, + # and to lock the mutex for the shortest possible amount of time. Otherwise, # all threads are fighting over this mutex and when it gets acquired it gets locked - # until a threads finishes downloading a gem, leaving the other threads waiting + # until a thread finishes downloading a gem, leaving the other threads waiting # doing nothing. def rubygems_gem_installer(spec, options) @gem_installers_mutex.synchronize { @gem_installers[spec.name] } || begin diff --git a/lib/bundler/templates/newgem/spec/newgem_spec.rb.tt b/lib/bundler/templates/newgem/spec/newgem_spec.rb.tt index dd39342f788590..7c6cde170b340e 100644 --- a/lib/bundler/templates/newgem/spec/newgem_spec.rb.tt +++ b/lib/bundler/templates/newgem/spec/newgem_spec.rb.tt @@ -9,7 +9,7 @@ RSpec.describe <%= config[:constant_name] %> do it "can call into Rust" do result = <%= config[:constant_name] %>.hello("world") - expect(result).to be("Hello earth, from Rust!") + expect(result).to eq("Hello world, from Rust!") end <%- else -%> it "does something useful" do diff --git a/lib/bundler/templates/newgem/test/minitest/test_newgem.rb.tt b/lib/bundler/templates/newgem/test/minitest/test_newgem.rb.tt index 9a859fa4d1eda2..844d3aff81ad1f 100644 --- a/lib/bundler/templates/newgem/test/minitest/test_newgem.rb.tt +++ b/lib/bundler/templates/newgem/test/minitest/test_newgem.rb.tt @@ -9,7 +9,7 @@ class <%= config[:minitest_constant_name] %> < Minitest::Test <%- if config[:ext] == 'rust' -%> def test_hello_world - assert_equal "Hello earth, from Rust!", <%= config[:constant_name] %>.hello("world") + assert_equal "Hello world, from Rust!", <%= config[:constant_name] %>.hello("world") end <%- else -%> def test_it_does_something_useful diff --git a/parse.y b/parse.y index 4f5c93c6006384..2c8be06373263e 100644 --- a/parse.y +++ b/parse.y @@ -1442,7 +1442,7 @@ static NODE *new_hash_pattern_tail(struct parser_params *p, NODE *kw_args, ID kw static rb_node_kw_arg_t *new_kw_arg(struct parser_params *p, NODE *k, const YYLTYPE *loc); static rb_node_args_t *args_with_numbered(struct parser_params*,rb_node_args_t*,int,ID); -static NODE* negate_lit(struct parser_params*, NODE*); +static NODE* negate_lit(struct parser_params*, NODE*,const YYLTYPE*); static void no_blockarg(struct parser_params*,NODE*); static NODE *ret_args(struct parser_params*,NODE*); static NODE *arg_blk_pass(NODE*,rb_node_block_pass_t*); @@ -6133,7 +6133,7 @@ numeric : simple_numeric | tUMINUS_NUM simple_numeric %prec tLOWEST { $$ = $2; - negate_lit(p, $$); + negate_lit(p, $$, &@$); /*% ripper: unary!(ID2VAL(idUMinus), $:2) %*/ } ; @@ -14358,7 +14358,7 @@ ret_args(struct parser_params *p, NODE *node) } static NODE* -negate_lit(struct parser_params *p, NODE* node) +negate_lit(struct parser_params *p, NODE* node, const YYLTYPE *loc) { switch (nd_type(node)) { case NODE_INTEGER: @@ -14374,6 +14374,7 @@ negate_lit(struct parser_params *p, NODE* node) RNODE_IMAGINARY(node)->minus = TRUE; break; } + node->nd_loc = *loc; return node; } diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index 9baf0464c8440c..b2a82caf017b91 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -478,7 +478,7 @@ update_git("foo", path: repo, tag: tag) install_gemfile <<-G - source "https://gem.repo1" + source "https://gem.repo1" git "#{repo}", :tag => #{tag.dump} do gem "foo" end @@ -1078,7 +1078,7 @@ expect(out).to eq("WIN") end - it "does not to a remote fetch if the revision is cached locally" do + it "does not do a remote fetch if the revision is cached locally" do build_git "foo" install_gemfile <<-G diff --git a/spec/bundler/install/gems/resolving_spec.rb b/spec/bundler/install/gems/resolving_spec.rb index e6a395a6696315..111d361aab9c4f 100644 --- a/spec/bundler/install/gems/resolving_spec.rb +++ b/spec/bundler/install/gems/resolving_spec.rb @@ -358,7 +358,7 @@ #{lockfile_platforms} DEPENDENCIES - parallel_tests + rubocop #{checksums} BUNDLED WITH #{Bundler::VERSION} diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb index fe0402cd09f509..2ba87c60802f75 100644 --- a/test/ruby/test_ast.rb +++ b/test/ruby/test_ast.rb @@ -1746,6 +1746,11 @@ def test_yield_locations assert_locations(node.children[-1].children[-1].children[-1].locations, [[1, 9, 1, 20], [1, 9, 1, 14], [1, 14, 1, 15], [1, 19, 1, 20]]) end + def test_negative_numeric_locations + node = ast_parse("-1") + assert_locations(node.children.last.locations, [[1, 0, 1, 2]]) + end + private def ast_parse(src, **options) begin @@ -1759,7 +1764,7 @@ def ast_parse(src, **options) def assert_locations(locations, expected) ary = locations.map {|loc| loc && [loc.first_lineno, loc.first_column, loc.last_lineno, loc.last_column] } - assert_equal(ary, expected) + assert_equal(expected, ary) end end end diff --git a/test/rubygems/test_gem_commands_push_command.rb b/test/rubygems/test_gem_commands_push_command.rb index 186ef68701e25b..ada95e89b4a85d 100644 --- a/test/rubygems/test_gem_commands_push_command.rb +++ b/test/rubygems/test_gem_commands_push_command.rb @@ -224,7 +224,7 @@ def test_execute_allowed_push_host spec.metadata["allowed_push_host"] = "https://privategemserver.example" end - @response = "Successfully registered gem: freewill (1.0.0)" + @response = "Successfully registered gem: freebird (1.0.1)" @fetcher.data["#{@spec.metadata["allowed_push_host"]}/api/v1/gems"] = HTTPResponseFactory.create(body: @response, code: 200, msg: "OK") @fetcher.data["#{Gem.host}/api/v1/gems"] = ["fail", 500, "Internal Server Error"] diff --git a/test/rubygems/test_gem_stub_specification.rb b/test/rubygems/test_gem_stub_specification.rb index f04b36dbf86cfd..6c07480c7fd4d6 100644 --- a/test/rubygems/test_gem_stub_specification.rb +++ b/test/rubygems/test_gem_stub_specification.rb @@ -221,7 +221,7 @@ def test_to_spec_with_other_specs_loaded_does_not_warn end def stub_with_version - spec = File.join @gemhome, "specifications", "stub_e-2.gemspec" + spec = File.join @gemhome, "specifications", "stub_v-with-version.gemspec" File.open spec, "w" do |io| io.write <<~STUB # -*- encoding: utf-8 -*- @@ -244,7 +244,7 @@ def stub_with_version end def stub_without_version - spec = File.join @gemhome, "specifications", "stub-2.gemspec" + spec = File.join @gemhome, "specifications", "stub_v-without-version.gemspec" File.open spec, "w" do |io| io.write <<~STUB # -*- encoding: utf-8 -*- diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 46b24e583b5885..5df020f4f56bc6 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -731,7 +731,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::GetIvar { self_val, id, ic, state: _ } => gen_getivar(jit, asm, opnd!(self_val), *id, *ic), Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), - &Insn::IsBlockParamModified { ep } => gen_is_block_param_modified(asm, opnd!(ep)), + &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), @@ -896,8 +896,7 @@ fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: } /// Returns 1 (as CBool) when VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set; returns 0 otherwise. -fn gen_is_block_param_modified(asm: &mut Assembler, ep: Opnd) -> Opnd { - let flags = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32))); +fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) } diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index dca1a402ca9e38..d57efdc698c031 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -447,6 +447,20 @@ fn test_getblockparamproxy_modified_nested_block() { assert_snapshot!(inspect("test { 1 }.call"), @"1"); } +#[test] +fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(assert_compiles("test { 2 }"), @"2"); +} + #[test] fn test_getblockparam() { eval(" diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 38c8044f8f6f48..6ea041d72f078c 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1564,6 +1564,7 @@ pub(crate) mod ids { name: _env_data_index_specval name: _ep_method_entry name: _ep_specval + name: _ep_flags name: _rbasic_flags name: RUBY_FL_FREEZE name: RUBY_ELTS_SHARED diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index db825598745410..9a09e8789cea97 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -531,6 +531,8 @@ pub enum SideExitReason { Interrupt, BlockParamProxyNotIseqOrIfunc, BlockParamProxyNotNil, + BlockParamProxyFallbackMiss, + BlockParamProxyProfileNotCovered, BlockParamWbRequired, StackOverflow, FixnumModByZero, @@ -923,9 +925,9 @@ pub enum Insn { StoreField { recv: InsnId, id: ID, offset: i32, val: InsnId }, WriteBarrier { recv: InsnId, val: InsnId }, - /// Check whether VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set in the environment flags. + /// Check whether VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set in the (already loaded) environment flags. /// Returns CBool (0/1). - IsBlockParamModified { ep: InsnId }, + IsBlockParamModified { flags: InsnId }, /// Get the block parameter as a Proc. GetBlockParam { level: u32, ep_offset: u32, state: InsnId }, /// Set a local variable in a higher scope or the heap @@ -1160,8 +1162,8 @@ macro_rules! for_each_operand_impl { Insn::IsBlockGiven { lep } => { $visit_one!(lep); } - Insn::IsBlockParamModified { ep } => { - $visit_one!(ep); + Insn::IsBlockParamModified { flags } => { + $visit_one!(flags); } Insn::CheckMatch { target, pattern, state, .. } => { $visit_one!(target); @@ -1586,7 +1588,7 @@ impl Insn { Insn::GetSpecialNumber { .. } => effects::Any, Insn::GetClassVar { .. } => effects::Any, Insn::SetClassVar { .. } => effects::Any, - Insn::IsBlockParamModified { .. } => effects::Any, + Insn::IsBlockParamModified { .. } => effects::Empty, Insn::GetBlockParam { .. } => effects::Any, Insn::Snapshot { .. } => effects::Empty, Insn::Jump(_) => effects::Any, @@ -2139,8 +2141,8 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::SetIvar { self_val, id, val, .. } => write!(f, "SetIvar {self_val}, :{}, {val}", id.contents_lossy()), Insn::GetGlobal { id, .. } => write!(f, "GetGlobal :{}", id.contents_lossy()), Insn::SetGlobal { id, val, .. } => write!(f, "SetGlobal :{}, {val}", id.contents_lossy()), - &Insn::IsBlockParamModified { ep } => { - write!(f, "IsBlockParamModified {ep}") + &Insn::IsBlockParamModified { flags } => { + write!(f, "IsBlockParamModified {flags}") }, &Insn::SetLocal { val, level, ep_offset } => { let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); @@ -2831,7 +2833,7 @@ impl Function { &GuardGreaterEq { left, right, reason, state } => GuardGreaterEq { left: find!(left), right: find!(right), reason, state }, &GuardLess { left, right, state } => GuardLess { left: find!(left), right: find!(right), state }, &IsBlockGiven { lep } => IsBlockGiven { lep: find!(lep) }, - &IsBlockParamModified { ep } => IsBlockParamModified { ep: find!(ep) }, + &IsBlockParamModified { flags } => IsBlockParamModified { flags: find!(flags) }, &GetBlockParam { level, ep_offset, state } => GetBlockParam { level, ep_offset, state: find!(state) }, &FixnumAdd { left, right, state } => FixnumAdd { left: find!(left), right: find!(right), state }, &FixnumSub { left, right, state } => FixnumSub { left: find!(left), right: find!(right), state }, @@ -3618,6 +3620,10 @@ impl Function { self.push_insn(block, Insn::LoadField { recv, id: ID!(_rbasic_flags), offset: RUBY_OFFSET_RBASIC_FLAGS, return_type: types::CUInt64 }) } + fn load_ep_flags(&mut self, block: BlockId, ep: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { recv: ep, id: ID!(_ep_flags), offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), return_type: types::CUInt64 }) + } + pub fn guard_not_frozen(&mut self, block: BlockId, recv: InsnId, state: InsnId) { let flags = self.load_rbasic_flags(block, recv); self.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(RUBY_FL_FREEZE as u64), mask_name: Some(ID!(RUBY_FL_FREEZE)), reason: SideExitReason::GuardNotFrozen, state }); @@ -4793,14 +4799,28 @@ impl Function { } let blockiseq = match send_block { - None | Some(BlockHandler::BlockArg) => unreachable!("went to reduce_send_without_block_to_ccall"), + Some(BlockHandler::BlockArg) => unreachable!("unsupported &block should have been filtered out"), Some(BlockHandler::BlockIseq(blockiseq)) => Some(blockiseq), + None => None, }; let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; // Find the `argc` (arity) of the C method, which describes the parameters it expects let cfunc_argc = unsafe { get_mct_argc(cfunc) }; let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + let name = unsafe { (*cme).called_id }; + + // Look up annotations + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + fun.count_not_annotated_cfunc(block, cme); + } + let props = props.unwrap_or_default(); + let return_type = props.return_type; + let elidable = match blockiseq { + Some(_) => false, // Don't consider cfuncs with block arguments as elidable for now + None => props.elidable, + }; match cfunc_argc { 0.. => { @@ -4826,20 +4846,48 @@ impl Function { fun.insn_types[recv.0] = fun.infer_type(recv); } - // Emit a call - let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.count(block, Counter::inline_cfunc_optimized_send_count); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); + } + } - let name = unsafe { (*cme).called_id }; + // Emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); + } let ccall = fun.push_insn(block, Insn::CCallWithFrame { cd, - cfunc, + cfunc: cfunc_ptr, recv, args, cme, name, state, - return_type: types::BasicObject, - elidable: false, + return_type, + elidable, block: blockiseq.map(BlockHandler::BlockIseq), }); fun.make_equal_to(send_insn_id, ccall); @@ -4864,215 +4912,8 @@ impl Function { fun.insn_types[recv.0] = fun.infer_type(recv); } - if get_option!(stats) { - fun.count_not_inlined_cfunc(block, cme); - } - - let ccall = fun.push_insn(block, Insn::CCallVariadic { - cfunc: cfunc_ptr, - recv, - args, - cme, - name: method_id, - state, - return_type: types::BasicObject, - elidable: false, - block: blockiseq.map(BlockHandler::BlockIseq), - }); - - fun.make_equal_to(send_insn_id, ccall); - Ok(()) - } - -2 => { - // (self, args_ruby_array) - fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); - Err(()) - } - _ => unreachable!("unknown cfunc kind: argc={argc}") - } - } - - // Try to reduce a Send insn with blockiseq: None to a CCall/CCallWithFrame - fn reduce_send_without_block_to_ccall( - fun: &mut Function, - block: BlockId, - self_type: Type, - send: Insn, - send_insn_id: InsnId, - ) -> Result<(), ()> { - let Insn::Send { mut recv, cd, args, state, .. } = send else { - return Err(()); - }; - - let call_info = unsafe { (*cd).ci }; - let argc = unsafe { vm_ci_argc(call_info) }; - let method_id = unsafe { rb_vm_ci_mid(call_info) }; - - // If we have info about the class of the receiver - let iseq_insn_idx = fun.frame_state(state).insn_idx; - let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { - ReceiverTypeResolution::StaticallyKnown { class } => (class, None), - ReceiverTypeResolution::Monomorphic { profiled_type } - | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), - ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), - }; - - // Do method lookup - let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; - if cme.is_null() { - fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); - return Err(()); - } - - // Filter for C methods - let mut def_type = unsafe { get_cme_def_type(cme) }; - while def_type == VM_METHOD_TYPE_ALIAS { - cme = unsafe { rb_aliased_callable_method_entry(cme) }; - def_type = unsafe { get_cme_def_type(cme) }; - } - if def_type != VM_METHOD_TYPE_CFUNC { - return Err(()); - } - - let ci_flags = unsafe { vm_ci_flag(call_info) }; - let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; - match (visibility, ci_flags & VM_CALL_FCALL != 0) { - (METHOD_VISI_PUBLIC, _) => {} - (METHOD_VISI_PRIVATE, true) => {} - (METHOD_VISI_PROTECTED, true) => {} - _ => { - fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedNeedPermission); - return Err(()); - } - } - - // Find the `argc` (arity) of the C method, which describes the parameters it expects - let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; - let cfunc_argc = unsafe { get_mct_argc(cfunc) }; - match cfunc_argc { - 0.. => { - // (self, arg0, arg1, ..., argc) form - // - // Bail on argc mismatch - if argc != cfunc_argc as u32 { - return Err(()); - } - - // Filter for simple call sites (i.e. no splats etc.) - if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { - // Only count features NOT already counted in type_specialize. - if !unspecializable_call_type(ci_flags) { - fun.count_complex_call_features(block, ci_flags); - } - fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); - return Err(()); - } - - // Check singleton class assumption first, before emitting other patchpoints - if !fun.assume_no_singleton_classes(block, recv_class, state) { - fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); - return Err(()); - } - - // Commit to the replacement. Put PatchPoint. - fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); - - let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); - if props.is_none() && get_option!(stats) { - fun.count_not_annotated_cfunc(block, cme); - } - let props = props.unwrap_or_default(); - - if let Some(profiled_type) = profiled_type { - // Guard receiver class - recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); - fun.insn_types[recv.0] = fun.infer_type(recv); - } - - // Try inlining the cfunc into HIR - let tmp_block = fun.new_block(u32::MAX); - if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { - // Copy contents of tmp_block to block - assert_ne!(block, tmp_block); - let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); - fun.blocks[block.0].insns.extend(insns); - fun.count(block, Counter::inline_cfunc_optimized_send_count); - fun.make_equal_to(send_insn_id, replacement); - if fun.type_of(replacement).bit_equal(types::Any) { - // Not set yet; infer type - fun.insn_types[replacement.0] = fun.infer_type(replacement); - } - fun.remove_block(tmp_block); - return Ok(()); - } - - // No inlining; emit a call - let cfunc = unsafe { get_mct_func(cfunc) }.cast(); - let name = unsafe { (*cme).called_id }; - let owner = unsafe { (*cme).owner }; - let return_type = props.return_type; - let elidable = props.elidable; - // Filter for a leaf and GC free function - if props.leaf && props.no_gc { - fun.count(block, Counter::inline_cfunc_optimized_send_count); - let ccall = fun.push_insn(block, Insn::CCall { cfunc, recv, args, name, owner, return_type, elidable }); - fun.make_equal_to(send_insn_id, ccall); - } else { - if get_option!(stats) { - fun.count_not_inlined_cfunc(block, cme); - } - let ccall = fun.push_insn(block, Insn::CCallWithFrame { - cd, - cfunc, - recv, - args, - cme, - name, - state, - return_type, - elidable, - block: None, - }); - fun.make_equal_to(send_insn_id, ccall); - } - - return Ok(()); - } - // Variadic method - -1 => { - // The method gets a pointer to the first argument - // func(int argc, VALUE *argv, VALUE recv) - let ci_flags = unsafe { vm_ci_flag(call_info) }; - if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { - // Only count features NOT already counted in type_specialize. - if !unspecializable_call_type(ci_flags) { - fun.count_complex_call_features(block, ci_flags); - } - fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); - return Err(()); - } else { - // Check singleton class assumption first, before emitting other patchpoints - if !fun.assume_no_singleton_classes(block, recv_class, state) { - fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); - return Err(()); - } - - fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); - - if let Some(profiled_type) = profiled_type { - // Guard receiver class - recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); - fun.insn_types[recv.0] = fun.infer_type(recv); - } - - let cfunc = unsafe { get_mct_func(cfunc) }.cast(); - let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); - if props.is_none() && get_option!(stats) { - fun.count_not_annotated_cfunc(block, cme); - } - let props = props.unwrap_or_default(); - - // Try inlining the cfunc into HIR + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { let tmp_block = fun.new_block(u32::MAX); if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { // Copy contents of tmp_block to block @@ -5089,40 +4930,43 @@ impl Function { return Ok(()); } - // No inlining; emit a call - if get_option!(stats) { - fun.count_not_inlined_cfunc(block, cme); + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); } - let return_type = props.return_type; - let elidable = props.elidable; - let name = unsafe { (*cme).called_id }; - let ccall = fun.push_insn(block, Insn::CCallVariadic { - cfunc, - recv, - args, - cme, - name, - state, - return_type, - elidable, - block: None, - }); + } - fun.make_equal_to(send_insn_id, ccall); - return Ok(()) + // No inlining; emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); } - // Fall through for complex cases (splat, kwargs, etc.) + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args, + cme, + name: method_id, + state, + return_type, + elidable, + block: blockiseq.map(BlockHandler::BlockIseq), + }); + + fun.make_equal_to(send_insn_id, ccall); + Ok(()) } -2 => { - // (self, args_ruby_array) parameter form - // Falling through for now - fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockCfuncArrayVariadic); + // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); + Err(()) } _ => unreachable!("unknown cfunc kind: argc={argc}") } - - Err(()) } for block in self.rpo() { @@ -5131,12 +4975,6 @@ impl Function { for insn_id in old_insns { let send = self.find(insn_id); match send { - send @ Insn::Send { recv, block: None, .. } => { - let recv_type = self.type_of(recv); - if reduce_send_without_block_to_ccall(self, block, recv_type, send, insn_id).is_ok() { - continue; - } - } send @ Insn::Send { recv, .. } => { let recv_type = self.type_of(recv); if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() { @@ -6165,7 +6003,6 @@ impl Function { | Insn::LoadField { .. } | Insn::GetConstantPath { .. } | Insn::IsBlockGiven { .. } - | Insn::IsBlockParamModified { .. } | Insn::GetGlobal { .. } | Insn::LoadPC | Insn::LoadSP @@ -6454,6 +6291,7 @@ impl Function { } Insn::RefineType { .. } => Ok(()), Insn::HasType { val, .. } => self.assert_subtype(insn_id, val, types::BasicObject), + Insn::IsBlockParamModified { flags } => self.assert_subtype(insn_id, flags, types::CUInt64), } } @@ -7575,17 +7413,31 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { }); } YARVINSN_getblockparamproxy => { + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + enum ProfiledBlockHandlerFamily { + Nil, + IseqOrIfunc, + } + impl ProfiledBlockHandlerFamily { + fn from_profiled_type(profiled_type: ProfiledType) -> Option { + let obj = profiled_type.class(); + if obj.nil_p() { + Some(Self::Nil) + } else if unsafe { + rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 + || rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 + } { + Some(Self::IseqOrIfunc) + } else { + None + } + } + } + let ep_offset = get_arg(pc, 0).as_u32(); let level = get_arg(pc, 1).as_u32(); let branch_insn_idx = exit_state.insn_idx as u32; - let profiled_block_type = if let Some([block_handler_distribution]) = profiles.payload.profile.get_operand_types(exit_state.insn_idx) { - let summary = TypeDistributionSummary::new(block_handler_distribution); - summary.is_monomorphic().then_some(summary.bucket(0).class()) - } else { - None - }; - // `getblockparamproxy` has two semantic paths: // - modified: return the already-materialized block local from EP // - unmodified: inspect the block handler and produce proxy/nil @@ -7596,7 +7448,8 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let join_local = if level == 0 { Some(fun.push_insn(join_block, Insn::Param)) } else { None }; let ep = fun.push_insn(block, Insn::GetEP { level }); - let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { ep }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); fun.push_insn(block, Insn::IfTrue { val: is_modified, target: BranchEdge { target: modified_block, args: vec![] }}); fun.push_insn(block, Insn::Jump(BranchEdge { target: unmodified_block, args: vec![] })); @@ -7611,30 +7464,136 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { // decide whether this path returns `nil` or `BlockParamProxy`. let block_handler = fun.push_insn(unmodified_block, Insn::LoadField { recv: ep, id: ID!(_env_data_index_specval), offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::CInt64 }); let original_local = if level == 0 { Some(state.getlocal(ep_offset)) } else { None }; + // `block_handler & 1 == 1` accepts both ISEQ (0b01) and ifunc + // (0b11) handlers. Keep a compile-time check that this shortcut + // does not accidentally accept symbol block handlers. + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + + let profiled_block_summary = profiles.payload.profile.get_operand_types(exit_state.insn_idx) + .and_then(|types| types.first()) + .map(TypeDistributionSummary::new); - match profiled_block_type { - Some(ty) if ty.nil_p() => { - fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: SideExitReason::BlockParamProxyNotNil, state: exit_id, recompile: None }); - let nil_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(Qnil) }); - let mut unmodified_args = vec![nil_val]; - if let Some(local) = original_local { unmodified_args.push(local); } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args: unmodified_args })); + let mut profiled_handlers = Vec::new(); + if let Some(summary) = profiled_block_summary.as_ref() { + if summary.is_monomorphic() || summary.is_polymorphic() || summary.is_skewed_polymorphic() { + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { + break; + } + if let Some(profiled_handler) = ProfiledBlockHandlerFamily::from_profiled_type(profiled_type) { + if !profiled_handlers.contains(&profiled_handler) { + profiled_handlers.push(profiled_handler); + } + } + } } - _ => { - // This handles two cases which are nearly identical + } + + match profiled_handlers.as_slice() { + // No supported profiled families. Keep the generic fallback iseq/ifunc fallback + // for sites we do not specialize, such as no-profile and megamorphic sites. + [] => { + // This handles two cases which are nearly identical. // Block handler is a tagged pointer. Look at the tag. // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 // So to check for either of those cases we can use: val & 0x1 == 0x1 - const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); // Bail out if the block handler is neither ISEQ nor ifunc - fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyNotIseqOrIfunc, state: exit_id }); + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyFallbackMiss, state: exit_id }); // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); - let mut unmodified_args = vec![proxy_val]; - if let Some(local) = original_local { unmodified_args.push(local); } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args: unmodified_args })); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + // A single supported profiled family. Emit a monomorphic fast path + [profiled_handler] => match profiled_handler { + ProfiledBlockHandlerFamily::Nil => { + fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: SideExitReason::BlockParamProxyNotNil, state: exit_id, recompile: None }); + let nil_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![nil_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyNotIseqOrIfunc, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + }, + // Multiple supported profiled families. Emit a polymorphic dispatch + _ => { + let profiled_blocks = profiled_handlers.iter() + .map(|&kind| (kind, fun.new_block(branch_insn_idx))) + .collect::>(); + + for &(kind, profiled_block) in &profiled_blocks { + match kind { + ProfiledBlockHandlerFamily::Nil => { + let none_handler = fun.push_insn(unmodified_block, Insn::Const { + val: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), + }); + let is_none = fun.push_insn(unmodified_block, Insn::IsBitEqual { + left: block_handler, + right: none_handler, + }); + fun.push_insn(unmodified_block, Insn::IfTrue { + val: is_none, + target: BranchEdge { target: profiled_block, args: vec![] }, + }); + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + let tag_mask = fun.push_insn(unmodified_block, Insn::Const { val: Const::CInt64(0x1) }); + let tag_bits = fun.push_insn(unmodified_block, Insn::IntAnd { + left: block_handler, + right: tag_mask, + }); + let is_iseq_or_ifunc = fun.push_insn(unmodified_block, Insn::IsBitEqual { + left: tag_bits, + right: tag_mask, + }); + fun.push_insn(unmodified_block, Insn::IfTrue { + val: is_iseq_or_ifunc, + target: BranchEdge { target: profiled_block, args: vec![] }, + }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + }, + } + } + + fun.push_insn(unmodified_block, Insn::SideExit { state: exit_id, reason: SideExitReason::BlockParamProxyProfileNotCovered, recompile: None }); } } @@ -7659,7 +7618,8 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let join_param = fun.push_insn(join_block, Insn::Param); let ep = fun.push_insn(block, Insn::GetEP { level }); - let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { ep }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); fun.push_insn(block, Insn::IfTrue { val: is_modified, diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 1364cf4632ba50..88ab8d30711147 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -1512,7 +1512,7 @@ mod hir_opt_tests { v4:BasicObject = LoadArg :self@0 Jump bb3(v4) bb3(v6:BasicObject): - v11:BasicObject = Send v6, :foo # SendFallbackReason: SendWithoutBlock: unsupported method type Null + v11:BasicObject = Send v6, :foo # SendFallbackReason: Send: unsupported method type Null CheckInterrupts Return v11 "); @@ -4729,15 +4729,16 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v17:CPtr = GetEP 0 - v18:CBool = IsBlockParamModified v17 - IfTrue v18, bb4() - v23:CInt64 = LoadField v17, :_env_data_index_specval@0x1001 - v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) - v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v25, v10) + v18:CUInt64 = LoadField v17, :_ep_flags@0x1001 + v19:CBool = IsBlockParamModified v18 + IfTrue v19, bb4() + v24:CInt64 = LoadField v17, :_env_data_index_specval@0x1002 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) bb4(): - v21:BasicObject = LoadField v17, :block@0x1010 - Jump bb6(v21, v21) + v22:BasicObject = LoadField v17, :block@0x1010 + Jump bb6(v22, v22) bb6(v15:BasicObject, v16:BasicObject): SideExit NoProfileSend recompile "); @@ -4769,25 +4770,27 @@ mod hir_opt_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): v18:CPtr = GetEP 0 - v19:CBool = IsBlockParamModified v18 - IfTrue v19, bb4() - v24:BasicObject = GetBlockParam :block, l0, EP@4 - Jump bb6(v24) + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + v25:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v25) bb4(): - v22:BasicObject = LoadField v18, :block@0x1001 - Jump bb6(v22) + v23:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v23) bb6(v17:BasicObject): - v32:CPtr = GetEP 0 - v33:CBool = IsBlockParamModified v32 - IfTrue v33, bb7() - v38:CInt64 = LoadField v32, :_env_data_index_specval@0x1002 - v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) - v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v40, v17) + v33:CPtr = GetEP 0 + v34:CUInt64 = LoadField v33, :_ep_flags@0x1001 + v35:CBool = IsBlockParamModified v34 + IfTrue v35, bb7() + v40:CInt64 = LoadField v33, :_env_data_index_specval@0x1003 + v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v42, v17) bb7(): - v36:BasicObject = LoadField v32, :block@0x1001 - Jump bb9(v36, v36) - bb9(v30:BasicObject, v31:BasicObject): + v38:BasicObject = LoadField v33, :block@0x1002 + Jump bb9(v38, v38) + bb9(v31:BasicObject, v32:BasicObject): SideExit NoProfileSend recompile "); } @@ -4816,29 +4819,87 @@ mod hir_opt_tests { Jump bb3(v5, v6) bb3(v8:BasicObject, v9:NilClass): v14:CPtr = GetEP 1 - v15:CBool = IsBlockParamModified v14 - IfTrue v15, bb4() - v20:BasicObject = GetBlockParam :block, l1, EP@3 - Jump bb6(v20) + v15:CUInt64 = LoadField v14, :_ep_flags@0x1000 + v16:CBool = IsBlockParamModified v15 + IfTrue v16, bb4() + v21:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v21) bb4(): - v18:BasicObject = LoadField v14, :block@0x1000 - Jump bb6(v18) + v19:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v19) bb6(v13:BasicObject): - v27:CPtr = GetEP 1 - v28:CBool = IsBlockParamModified v27 - IfTrue v28, bb7() - v33:CInt64 = LoadField v27, :_env_data_index_specval@0x1001 - v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) - v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v35) + v28:CPtr = GetEP 1 + v29:CUInt64 = LoadField v28, :_ep_flags@0x1000 + v30:CBool = IsBlockParamModified v29 + IfTrue v30, bb7() + v35:CInt64 = LoadField v28, :_env_data_index_specval@0x1002 + v36:CInt64 = GuardAnyBitSet v35, CUInt64(1) + v37:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v37) bb7(): - v31:BasicObject = LoadField v27, :block@0x1000 - Jump bb9(v31) - bb9(v26:BasicObject): + v33:BasicObject = LoadField v28, :block@0x1001 + Jump bb9(v33) + bb9(v27:BasicObject): SideExit NoProfileSend recompile "); } + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + v25:CInt64 = LoadField v18, :_env_data_index_specval@0x1002 + v26:CInt64[1] = Const CInt64(1) + v27:CInt64 = IntAnd v25, v26 + v28:CBool = IsBitEqual v27, v26 + IfTrue v28, bb7() + v32:CInt64[0] = Const CInt64(0) + v33:CBool = IsBitEqual v25, v32 + IfTrue v33, bb8() + SideExit BlockParamProxyProfileNotCovered + bb4(): + v23:BasicObject = LoadField v18, :block@0x1003 + Jump bb6(v23, v23) + bb7(): + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v10) + bb8(): + v35:NilClass = Const Value(nil) + Jump bb6(v35, v10) + bb6(v16:BasicObject, v17:BasicObject): + v39:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v39 + "); + } + #[test] fn test_getblockparam() { eval(" @@ -4859,13 +4920,14 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v15:CPtr = GetEP 0 - v16:CBool = IsBlockParamModified v15 - IfTrue v16, bb4() - v21:BasicObject = GetBlockParam :block, l0, EP@3 - Jump bb6(v21) + v16:CUInt64 = LoadField v15, :_ep_flags@0x1001 + v17:CBool = IsBlockParamModified v16 + IfTrue v17, bb4() + v22:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v22) bb4(): - v19:BasicObject = LoadField v15, :block@0x1001 - Jump bb6(v19) + v20:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v20) bb6(v14:BasicObject): CheckInterrupts Return v14 @@ -4893,13 +4955,14 @@ mod hir_opt_tests { Jump bb3(v4) bb3(v6:BasicObject): v11:CPtr = GetEP 1 - v12:CBool = IsBlockParamModified v11 - IfTrue v12, bb4() - v17:BasicObject = GetBlockParam :block, l1, EP@3 - Jump bb6(v17) + v12:CUInt64 = LoadField v11, :_ep_flags@0x1000 + v13:CBool = IsBlockParamModified v12 + IfTrue v13, bb4() + v18:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v18) bb4(): - v15:BasicObject = LoadField v11, :block@0x1000 - Jump bb6(v15) + v16:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v16) bb6(v10:BasicObject): CheckInterrupts Return v10 @@ -8149,19 +8212,20 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v14:ArrayExact = NewArray v18:CPtr = GetEP 0 - v19:CBool = IsBlockParamModified v18 - IfTrue v19, bb4() - v24:CInt64 = LoadField v18, :_env_data_index_specval@0x1001 - v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) - v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v26, v10) + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + v25:CInt64 = LoadField v18, :_env_data_index_specval@0x1002 + v26:CInt64 = GuardAnyBitSet v25, CUInt64(1) + v27:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v27, v10) bb4(): - v22:BasicObject = LoadField v18, :block@0x1010 - Jump bb6(v22, v22) + v23:BasicObject = LoadField v18, :block@0x1010 + Jump bb6(v23, v23) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + v30:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing CheckInterrupts - Return v29 + Return v30 "); } @@ -8187,19 +8251,20 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v14:ArrayExact = NewArray v18:CPtr = GetEP 0 - v19:CBool = IsBlockParamModified v18 - IfTrue v19, bb4() - v24:CInt64 = LoadField v18, :_env_data_index_specval@0x1001 - v25:CInt64[0] = GuardBitEquals v24, CInt64(0) - v26:NilClass = Const Value(nil) - Jump bb6(v26, v10) + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + v25:CInt64 = LoadField v18, :_env_data_index_specval@0x1002 + v26:CInt64[0] = GuardBitEquals v25, CInt64(0) + v27:NilClass = Const Value(nil) + Jump bb6(v27, v10) bb4(): - v22:BasicObject = LoadField v18, :block@0x1002 - Jump bb6(v22, v22) + v23:BasicObject = LoadField v18, :block@0x1003 + Jump bb6(v23, v23) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + v30:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing CheckInterrupts - Return v29 + Return v30 "); } @@ -8226,19 +8291,20 @@ mod hir_opt_tests { bb3(v6:BasicObject): v10:ArrayExact = NewArray v13:CPtr = GetEP 1 - v14:CBool = IsBlockParamModified v13 - IfTrue v14, bb4() - v19:CInt64 = LoadField v13, :_env_data_index_specval@0x1000 - v20:CInt64 = GuardAnyBitSet v19, CUInt64(1) - v21:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v21) + v14:CUInt64 = LoadField v13, :_ep_flags@0x1000 + v15:CBool = IsBlockParamModified v14 + IfTrue v15, bb4() + v20:CInt64 = LoadField v13, :_env_data_index_specval@0x1001 + v21:CInt64 = GuardAnyBitSet v20, CUInt64(1) + v22:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v22) bb4(): - v17:BasicObject = LoadField v13, :block@0x1010 - Jump bb6(v17) + v18:BasicObject = LoadField v13, :block@0x1010 + Jump bb6(v18) bb6(v12:BasicObject): - v24:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Complex argument passing + v25:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Complex argument passing CheckInterrupts - Return v24 + Return v25 "); } @@ -11713,19 +11779,20 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v14:ArrayExact = NewArray v18:CPtr = GetEP 0 - v19:CBool = IsBlockParamModified v18 - IfTrue v19, bb4() - v24:CInt64 = LoadField v18, :_env_data_index_specval@0x1001 - v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) - v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v26, v10) + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + v25:CInt64 = LoadField v18, :_env_data_index_specval@0x1002 + v26:CInt64 = GuardAnyBitSet v25, CUInt64(1) + v27:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v27, v10) bb4(): - v22:BasicObject = LoadField v18, :block@0x1010 - Jump bb6(v22, v22) + v23:BasicObject = LoadField v18, :block@0x1010 + Jump bb6(v23, v23) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + v30:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing CheckInterrupts - Return v29 + Return v30 "); } @@ -13845,7 +13912,7 @@ mod hir_opt_tests { PatchPoint SingleRactorMode PatchPoint StableConstantNames(0x1000, Obj) v21:BasicObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v12:BasicObject = Send v21, :initialize # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + v12:BasicObject = Send v21, :initialize # SendFallbackReason: Send: method private or protected and no FCALL CheckInterrupts Return v12 "); diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 542e0c0614909a..3358d7573593b7 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -2467,17 +2467,18 @@ pub(crate) mod hir_build_tests { v29:ArrayExact = ToArray v19 PatchPoint NoEPEscape(test) v36:CPtr = GetEP 0 - v37:CBool = IsBlockParamModified v36 - IfTrue v37, bb4() + v37:CUInt64 = LoadField v36, :_ep_flags@0x1004 + v38:CBool = IsBlockParamModified v37 + IfTrue v38, bb4() Jump bb5() bb4(): - v40:BasicObject = LoadField v36, :&@0x1004 - Jump bb6(v40, v40) + v41:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v41, v41) bb5(): - v42:CInt64 = LoadField v36, :_env_data_index_specval@0x1005 - v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) - v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v44, v21) + v43:CInt64 = LoadField v36, :_env_data_index_specval@0x1006 + v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) + v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v45, v21) bb6(v34:BasicObject, v35:BasicObject): SideExit SplatKwNotProfiled "); @@ -3390,15 +3391,16 @@ pub(crate) mod hir_build_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v15:CPtr = GetEP 0 - v16:CBool = IsBlockParamModified v15 - IfTrue v16, bb4() + v16:CUInt64 = LoadField v15, :_ep_flags@0x1001 + v17:CBool = IsBlockParamModified v16 + IfTrue v17, bb4() Jump bb5() bb4(): - v19:BasicObject = LoadField v15, :block@0x1001 - Jump bb6(v19) + v20:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v20) bb5(): - v21:BasicObject = GetBlockParam :block, l0, EP@3 - Jump bb6(v21) + v22:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v22) bb6(v14:BasicObject): CheckInterrupts Return v14 @@ -3426,21 +3428,22 @@ pub(crate) mod hir_build_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v17:CPtr = GetEP 0 - v18:CBool = IsBlockParamModified v17 - IfTrue v18, bb4() + v18:CUInt64 = LoadField v17, :_ep_flags@0x1001 + v19:CBool = IsBlockParamModified v18 + IfTrue v19, bb4() Jump bb5() bb4(): - v21:BasicObject = LoadField v17, :block@0x1001 - Jump bb6(v21, v21) + v22:BasicObject = LoadField v17, :block@0x1002 + Jump bb6(v22, v22) bb5(): - v23:CInt64 = LoadField v17, :_env_data_index_specval@0x1002 - v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) - v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v25, v10) + v24:CInt64 = LoadField v17, :_env_data_index_specval@0x1003 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) bb6(v15:BasicObject, v16:BasicObject): - v28:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) + v29:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v28 + Return v29 "); } @@ -3470,32 +3473,34 @@ pub(crate) mod hir_build_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): v18:CPtr = GetEP 0 - v19:CBool = IsBlockParamModified v18 - IfTrue v19, bb4() + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() Jump bb5() bb4(): - v22:BasicObject = LoadField v18, :block@0x1001 - Jump bb6(v22) + v23:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v23) bb5(): - v24:BasicObject = GetBlockParam :block, l0, EP@4 - Jump bb6(v24) + v25:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v25) bb6(v17:BasicObject): - v32:CPtr = GetEP 0 - v33:CBool = IsBlockParamModified v32 - IfTrue v33, bb7() + v33:CPtr = GetEP 0 + v34:CUInt64 = LoadField v33, :_ep_flags@0x1001 + v35:CBool = IsBlockParamModified v34 + IfTrue v35, bb7() Jump bb8() bb7(): - v36:BasicObject = LoadField v32, :block@0x1001 - Jump bb9(v36, v36) + v38:BasicObject = LoadField v33, :block@0x1002 + Jump bb9(v38, v38) bb8(): - v38:CInt64 = LoadField v32, :_env_data_index_specval@0x1002 - v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) - v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v40, v17) - bb9(v30:BasicObject, v31:BasicObject): - v43:BasicObject = Send v11, &block, :tap, v30 # SendFallbackReason: Uncategorized(send) + v40:CInt64 = LoadField v33, :_env_data_index_specval@0x1003 + v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v42, v17) + bb9(v31:BasicObject, v32:BasicObject): + v45:BasicObject = Send v11, &block, :tap, v31 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v43 + Return v45 "); } @@ -3523,32 +3528,92 @@ pub(crate) mod hir_build_tests { Jump bb3(v5, v6) bb3(v8:BasicObject, v9:NilClass): v14:CPtr = GetEP 1 - v15:CBool = IsBlockParamModified v14 - IfTrue v15, bb4() + v15:CUInt64 = LoadField v14, :_ep_flags@0x1000 + v16:CBool = IsBlockParamModified v15 + IfTrue v16, bb4() Jump bb5() bb4(): - v18:BasicObject = LoadField v14, :block@0x1000 - Jump bb6(v18) + v19:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v19) bb5(): - v20:BasicObject = GetBlockParam :block, l1, EP@3 - Jump bb6(v20) + v21:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v21) bb6(v13:BasicObject): - v27:CPtr = GetEP 1 - v28:CBool = IsBlockParamModified v27 - IfTrue v28, bb7() + v28:CPtr = GetEP 1 + v29:CUInt64 = LoadField v28, :_ep_flags@0x1000 + v30:CBool = IsBlockParamModified v29 + IfTrue v30, bb7() Jump bb8() bb7(): - v31:BasicObject = LoadField v27, :block@0x1000 - Jump bb9(v31) + v33:BasicObject = LoadField v28, :block@0x1001 + Jump bb9(v33) bb8(): - v33:CInt64 = LoadField v27, :_env_data_index_specval@0x1001 - v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) - v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v35) - bb9(v26:BasicObject): - v38:BasicObject = Send v8, &block, :tap, v26 # SendFallbackReason: Uncategorized(send) + v35:CInt64 = LoadField v28, :_env_data_index_specval@0x1002 + v36:CInt64 = GuardAnyBitSet v35, CUInt64(1) + v37:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v37) + bb9(v27:BasicObject): + v40:BasicObject = Send v8, &block, :tap, v27 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v38 + Return v40 + "); + } + + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :_ep_flags@0x1001 + v20:CBool = IsBlockParamModified v19 + IfTrue v20, bb4() + Jump bb5() + bb4(): + v23:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v23, v23) + bb5(): + v25:CInt64 = LoadField v18, :_env_data_index_specval@0x1003 + v26:CInt64[1] = Const CInt64(1) + v27:CInt64 = IntAnd v25, v26 + v28:CBool = IsBitEqual v27, v26 + IfTrue v28, bb7() + v32:CInt64[0] = Const CInt64(0) + v33:CBool = IsBitEqual v25, v32 + IfTrue v33, bb8() + SideExit BlockParamProxyProfileNotCovered + bb7(): + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v10) + bb8(): + v35:NilClass = Const Value(nil) + Jump bb6(v35, v10) + bb6(v16:BasicObject, v17:BasicObject): + v39:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v39 "); } @@ -3573,15 +3638,16 @@ pub(crate) mod hir_build_tests { Jump bb3(v4) bb3(v6:BasicObject): v11:CPtr = GetEP 1 - v12:CBool = IsBlockParamModified v11 - IfTrue v12, bb4() + v12:CUInt64 = LoadField v11, :_ep_flags@0x1000 + v13:CBool = IsBlockParamModified v12 + IfTrue v13, bb4() Jump bb5() bb4(): - v15:BasicObject = LoadField v11, :block@0x1000 - Jump bb6(v15) + v16:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v16) bb5(): - v17:BasicObject = GetBlockParam :block, l1, EP@3 - Jump bb6(v17) + v18:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v18) bb6(v10:BasicObject): CheckInterrupts Return v10 @@ -3678,17 +3744,18 @@ pub(crate) mod hir_build_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): v21:CPtr = GetEP 0 - v22:CBool = IsBlockParamModified v21 - IfTrue v22, bb4() + v22:CUInt64 = LoadField v21, :_ep_flags@0x1002 + v23:CBool = IsBlockParamModified v22 + IfTrue v23, bb4() Jump bb5() bb4(): - v25:BasicObject = LoadField v21, :b@0x1002 - Jump bb6(v25, v25) + v26:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v26, v26) bb5(): - v27:CInt64 = LoadField v21, :_env_data_index_specval@0x1003 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64 = LoadField v21, :_env_data_index_specval@0x1004 + v29:CInt64 = GuardAnyBitSet v28, CUInt64(1) + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v13) bb6(v19:BasicObject, v20:BasicObject): SideExit SplatKwNotProfiled "); @@ -3727,22 +3794,23 @@ pub(crate) mod hir_build_tests { v29:ArrayExact = ToArray v19 PatchPoint NoEPEscape(test) v36:CPtr = GetEP 0 - v37:CBool = IsBlockParamModified v36 - IfTrue v37, bb4() + v37:CUInt64 = LoadField v36, :_ep_flags@0x1004 + v38:CBool = IsBlockParamModified v37 + IfTrue v38, bb4() Jump bb5() bb4(): - v40:BasicObject = LoadField v36, :&@0x1004 - Jump bb6(v40, v40) + v41:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v41, v41) bb5(): - v42:CInt64 = LoadField v36, :_env_data_index_specval@0x1005 - v43:CInt64[0] = GuardBitEquals v42, CInt64(0) - v44:NilClass = Const Value(nil) - Jump bb6(v44, v21) + v43:CInt64 = LoadField v36, :_env_data_index_specval@0x1006 + v44:CInt64[0] = GuardBitEquals v43, CInt64(0) + v45:NilClass = Const Value(nil) + Jump bb6(v45, v21) bb6(v34:BasicObject, v35:BasicObject): - v47:NilClass = GuardType v20, NilClass - v49:BasicObject = Send v17, &block, :foo, v18, v29, v47, v34 # SendFallbackReason: Uncategorized(send) + v48:NilClass = GuardType v20, NilClass + v50:BasicObject = Send v17, &block, :foo, v18, v29, v48, v34 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v49 + Return v50 "); } @@ -3771,22 +3839,23 @@ pub(crate) mod hir_build_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): v21:CPtr = GetEP 0 - v22:CBool = IsBlockParamModified v21 - IfTrue v22, bb4() + v22:CUInt64 = LoadField v21, :_ep_flags@0x1002 + v23:CBool = IsBlockParamModified v22 + IfTrue v23, bb4() Jump bb5() bb4(): - v25:BasicObject = LoadField v21, :b@0x1002 - Jump bb6(v25, v25) + v26:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v26, v26) bb5(): - v27:CInt64 = LoadField v21, :_env_data_index_specval@0x1003 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64 = LoadField v21, :_env_data_index_specval@0x1004 + v29:CInt64 = GuardAnyBitSet v28, CUInt64(1) + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v13) bb6(v19:BasicObject, v20:BasicObject): - v32:HashExact = GuardType v12, HashExact - v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + v33:HashExact = GuardType v12, HashExact + v35:BasicObject = Send v11, &block, :foo, v33, v19 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v34 + Return v35 "); } @@ -3815,22 +3884,23 @@ pub(crate) mod hir_build_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): v21:CPtr = GetEP 0 - v22:CBool = IsBlockParamModified v21 - IfTrue v22, bb4() + v22:CUInt64 = LoadField v21, :_ep_flags@0x1002 + v23:CBool = IsBlockParamModified v22 + IfTrue v23, bb4() Jump bb5() bb4(): - v25:BasicObject = LoadField v21, :b@0x1002 - Jump bb6(v25, v25) + v26:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v26, v26) bb5(): - v27:CInt64 = LoadField v21, :_env_data_index_specval@0x1003 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64 = LoadField v21, :_env_data_index_specval@0x1004 + v29:CInt64 = GuardAnyBitSet v28, CUInt64(1) + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v13) bb6(v19:BasicObject, v20:BasicObject): - v32:HashExact = GuardType v12, HashExact - v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + v33:HashExact = GuardType v12, HashExact + v35:BasicObject = Send v11, &block, :foo, v33, v19 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v34 + Return v35 "); } @@ -3869,17 +3939,18 @@ pub(crate) mod hir_build_tests { v29:ArrayExact = ToArray v19 PatchPoint NoEPEscape(test) v36:CPtr = GetEP 0 - v37:CBool = IsBlockParamModified v36 - IfTrue v37, bb4() + v37:CUInt64 = LoadField v36, :_ep_flags@0x1004 + v38:CBool = IsBlockParamModified v37 + IfTrue v38, bb4() Jump bb5() bb4(): - v40:BasicObject = LoadField v36, :&@0x1004 - Jump bb6(v40, v40) + v41:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v41, v41) bb5(): - v42:CInt64 = LoadField v36, :_env_data_index_specval@0x1005 - v43:CInt64[0] = GuardBitEquals v42, CInt64(0) - v44:NilClass = Const Value(nil) - Jump bb6(v44, v21) + v43:CInt64 = LoadField v36, :_env_data_index_specval@0x1006 + v44:CInt64[0] = GuardBitEquals v43, CInt64(0) + v45:NilClass = Const Value(nil) + Jump bb6(v45, v21) bb6(v34:BasicObject, v35:BasicObject): SideExit SplatKwPolymorphic "); @@ -3912,17 +3983,18 @@ pub(crate) mod hir_build_tests { Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): v21:CPtr = GetEP 0 - v22:CBool = IsBlockParamModified v21 - IfTrue v22, bb4() + v22:CUInt64 = LoadField v21, :_ep_flags@0x1002 + v23:CBool = IsBlockParamModified v22 + IfTrue v23, bb4() Jump bb5() bb4(): - v25:BasicObject = LoadField v21, :block@0x1002 - Jump bb6(v25, v25) + v26:BasicObject = LoadField v21, :block@0x1003 + Jump bb6(v26, v26) bb5(): - v27:CInt64 = LoadField v21, :_env_data_index_specval@0x1003 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64 = LoadField v21, :_env_data_index_specval@0x1004 + v29:CInt64 = GuardAnyBitSet v28, CUInt64(1) + v30:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v30, v13) bb6(v19:BasicObject, v20:BasicObject): SideExit SplatKwNotNilOrHash "); @@ -4602,30 +4674,31 @@ pub(crate) mod hir_build_tests { v27:BasicObject = InvokeBuiltin dir_s_open, v18, v19, v20 PatchPoint NoEPEscape(open) v35:CPtr = GetEP 0 - v36:CBool = IsBlockParamModified v35 - IfTrue v36, bb5() + v36:CUInt64 = LoadField v35, :_ep_flags@0x1004 + v37:CBool = IsBlockParamModified v36 + IfTrue v37, bb5() Jump bb6() bb5(): - v39:BasicObject = LoadField v35, :block@0x1004 - Jump bb7(v39, v39) + v40:BasicObject = LoadField v35, :block@0x1005 + Jump bb7(v40, v40) bb6(): - v41:CInt64 = LoadField v35, :_env_data_index_specval@0x1005 - v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) - v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb7(v43, v22) + v42:CInt64 = LoadField v35, :_env_data_index_specval@0x1006 + v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb7(v44, v22) bb7(v33:BasicObject, v34:BasicObject): CheckInterrupts - v47:CBool = Test v33 - v48:Falsy = RefineType v33, Falsy - IfFalse v47, bb4(v18, v19, v20, v21, v34, v27) - v50:Truthy = RefineType v33, Truthy - v54:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized - v57:BasicObject = InvokeBuiltin dir_s_close, v18, v27 + v48:CBool = Test v33 + v49:Falsy = RefineType v33, Falsy + IfFalse v48, bb4(v18, v19, v20, v21, v34, v27) + v51:Truthy = RefineType v33, Truthy + v55:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + v58:BasicObject = InvokeBuiltin dir_s_close, v18, v27 CheckInterrupts - Return v54 - bb4(v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject): + Return v55 + bb4(v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject, v69:BasicObject): CheckInterrupts - Return v68 + Return v69 "); } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 2404615902074a..38d69c75339ea6 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -228,6 +228,8 @@ make_counters! { exit_stackoverflow, exit_block_param_proxy_not_iseq_or_ifunc, exit_block_param_proxy_not_nil, + exit_block_param_proxy_fallback_miss, + exit_block_param_proxy_profile_not_covered, exit_block_param_wb_required, exit_too_many_keyword_parameters, exit_no_profile_send, @@ -611,6 +613,8 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { StackOverflow => exit_stackoverflow, BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, BlockParamProxyNotNil => exit_block_param_proxy_not_nil, + BlockParamProxyFallbackMiss => exit_block_param_proxy_fallback_miss, + BlockParamProxyProfileNotCovered => exit_block_param_proxy_profile_not_covered, BlockParamWbRequired => exit_block_param_wb_required, TooManyKeywordParameters => exit_too_many_keyword_parameters, SplatKwNotNilOrHash => exit_splatkw_not_nil_or_hash,