diff --git a/array.c b/array.c index 2e0cfe5b03f16d..b6344fbdca7677 100644 --- a/array.c +++ b/array.c @@ -2944,13 +2944,20 @@ rb_zjit_array_dup_can_fastpath(VALUE ary, size_t *alloc_size_out, VALUE *flags_o if (len > embed_capa) return false; + *alloc_size_out = sizeof(struct RArray); + *flags_out = T_ARRAY | RARRAY_EMBED_FLAG | ((VALUE)len << RARRAY_EMBED_LEN_SHIFT); + *len_out = len; + return true; +} + +void +rb_zjit_array_new_fastpath(size_t *alloc_size_out, VALUE *flags_out) +{ size_t size = sizeof(struct RArray); shape_id_t shape_id = rb_shape_transition_slot_size(ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, rb_gc_size_slot_size(size)); *alloc_size_out = size; - *flags_out = T_ARRAY | RARRAY_EMBED_FLAG | ((VALUE)len << RARRAY_EMBED_LEN_SHIFT) | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); - *len_out = len; - return true; + *flags_out = T_ARRAY | RARRAY_EMBED_FLAG | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); } #endif diff --git a/gc.c b/gc.c index af259e06bb599c..43788bc950c584 100644 --- a/gc.c +++ b/gc.c @@ -1252,7 +1252,7 @@ rb_class_allocate_instance(VALUE klass) #if USE_ZJIT bool -rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, shape_id_t *shape_id_out) +rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, VALUE *flags_out) { uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass); @@ -1261,12 +1261,17 @@ rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, shape_id return false; } - size_t size = robject_embedded_size(index_tbl_num_entries); - *size_out = size; - *shape_id_out = rb_shape_transition_slot_size(rb_shape_transition_robject(0), - rb_gc_size_slot_size(size)); + *size_out = robject_embedded_size(index_tbl_num_entries); + *flags_out = T_OBJECT | rb_shape_transition_robject(0); + return true; } + +bool +rb_zjit_newobj_hook_enabled_p(void) +{ + return rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ); +} #endif void @@ -3773,7 +3778,7 @@ rb_gc_ractor_cache_free(void *cache) bool rb_gc_zjit_new_obj_fastpath(size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath) { -#if RACTOR_CHECK_MODE || defined(RUBY_ASAN_ENABLED) +#if defined(RUBY_ASAN_ENABLED) (void)rb_gc_impl_zjit_new_obj_fastpath; return false; #else diff --git a/gc/default/default.c b/gc/default/default.c index bc44e57fe7922b..441c118c5bd9c2 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -5418,6 +5418,11 @@ init_mark_stack(mark_stack_t *stack) /* Marking */ +ALWAYS_INLINE(static int gc_mark_set(rb_objspace_t *objspace, VALUE obj)); +ALWAYS_INLINE(static void gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj)); +ALWAYS_INLINE(static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)); +ALWAYS_INLINE(static void gc_aging(rb_objspace_t *objspace, VALUE obj)); +ALWAYS_INLINE(static void gc_grey(rb_objspace_t *objspace, VALUE obj)); static void rgengc_check_relation(rb_objspace_t *objspace, VALUE obj) { diff --git a/hash.c b/hash.c index 8799bb9b887c4d..c01c6dc84af1fc 100644 --- a/hash.c +++ b/hash.c @@ -1467,12 +1467,8 @@ hash_alloc(VALUE klass) size_t rb_zjit_hash_new_size(VALUE *flags_out) { - size_t size = hash_slot_size(sizeof(st_table) > sizeof(ar_table)); - // mimic rb_newobj() - shape_id_t shape_id = rb_shape_transition_slot_size(ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, - rb_gc_size_slot_size(size)); - *flags_out = T_HASH | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); - return size; + *flags_out = T_HASH; + return hash_slot_size(sizeof(st_table) > sizeof(ar_table)); } #endif diff --git a/ractor.c b/ractor.c index f8fbe3e380d2e9..d027ccb145069b 100644 --- a/ractor.c +++ b/ractor.c @@ -268,10 +268,6 @@ ractor_mark_unshareable_parts(rb_ractor_t *r) ccan_list_for_each(&r->threads.set, th, lt_node) { VM_ASSERT(th != NULL); rb_gc_mark(th->self); - /* Mark the EC directly: the stack must stay alive even in windows where - * the Thread wrapper's own mark has not been traversed yet (mid-creation, - * teardown). */ - if (th->ec) rb_execution_context_mark(th->ec); /* A thread's ec lives inside the root fiber struct and is freed with that * fiber's wrapper object, so keep the fiber wrappers alive from here too. */ @@ -279,9 +275,15 @@ ractor_mark_unshareable_parts(rb_ractor_t *r) VALUE root_fiber_self = rb_fiberptr_self(th->root_fiber); if (root_fiber_self) rb_gc_mark(root_fiber_self); } - if (th->ec && th->ec->fiber_ptr) { - VALUE fiber_self = rb_fiberptr_self(th->ec->fiber_ptr); - if (fiber_self) rb_gc_mark(fiber_self); + /* The ec sits inside its fiber, so marking that fiber's wrapper scans the ec + * as well. Only when there is no wrapper yet (mid-creation, teardown) does + * the ec need marking of its own. */ + VALUE ec_fiber_self = (th->ec && th->ec->fiber_ptr) ? rb_fiberptr_self(th->ec->fiber_ptr) : 0; + if (ec_fiber_self) { + rb_gc_mark(ec_fiber_self); + } + else if (th->ec) { + rb_execution_context_mark(th->ec); } /* Root the thread's remaining possessions directly as well; thgroup in diff --git a/range.c b/range.c index 31ef3ec1fe6524..9dbe57b0ef2d54 100644 --- a/range.c +++ b/range.c @@ -87,20 +87,13 @@ void rb_zjit_range_new_fastpath(bool exclude_end, size_t *alloc_size_out, VALUE *flags_out) { const long len = 2; - size_t size = offsetof(struct RStruct, as.ary) + (sizeof(VALUE) * len); + *alloc_size_out = offsetof(struct RStruct, as.ary) + (sizeof(VALUE) * len); if (RCLASS_MAX_IV_COUNT(rb_cRange) > 0) { - size += sizeof(VALUE); + *alloc_size_out += sizeof(VALUE); } - VALUE flags = T_STRUCT | (len << RSTRUCT_EMBED_LEN_SHIFT) | RANGE_FL_INIT | FL_FREEZE; - if (exclude_end) flags |= RANGE_FL_EXCL; - - shape_id_t shape_id = rb_shape_transition_slot_size(ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_EXTENDED, - rb_gc_size_slot_size(size)); - shape_id = rb_shape_transition_frozen(shape_id); - - *alloc_size_out = size; - *flags_out = flags | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); + *flags_out = T_STRUCT | (len << RSTRUCT_EMBED_LEN_SHIFT) | RANGE_FL_INIT | FL_FREEZE; + if (exclude_end) *flags_out |= RANGE_FL_EXCL; } #endif diff --git a/spec/mspec/lib/mspec/utils/deprecate.rb b/spec/mspec/lib/mspec/utils/deprecate.rb index 1db843b329200c..f5dbc2798e7254 100644 --- a/spec/mspec/lib/mspec/utils/deprecate.rb +++ b/spec/mspec/lib/mspec/utils/deprecate.rb @@ -1,6 +1,10 @@ module MSpec + FATAL_DEPRECATION = ENV['MSPEC_FATAL_DEPRECATION'] + def self.deprecate(what, replacement) user_caller = caller.find { |line| !line.include?('lib/mspec') } - $stderr.puts "\n#{what} is deprecated, use #{replacement} instead.\nfrom #{user_caller}" + message = "\n#{what} is deprecated, use #{replacement} instead.\nfrom #{user_caller}" + $stderr.puts message + raise SpecExpectationNotMetError, message if FATAL_DEPRECATION end end diff --git a/spec/ruby/core/array/sort_spec.rb b/spec/ruby/core/array/sort_spec.rb index 27300c338530bb..a3a382078240fa 100644 --- a/spec/ruby/core/array/sort_spec.rb +++ b/spec/ruby/core/array/sort_spec.rb @@ -112,20 +112,16 @@ end it "uses the sign of Integer block results as the sort result" do + ruby_exe(<<~RUBY).should == "[-4, 1, 2, 5, 7, 10, 12]\n" a = [1, 2, 5, 10, 7, -4, 12] - begin - class Integer - alias old_spaceship <=> - def <=>(other) - raise - end - end - a.sort {|n, m| (n - m) * (2 ** 200)}.should == [-4, 1, 2, 5, 7, 10, 12] - ensure - class Integer - alias <=> old_spaceship + class Integer + alias old_spaceship <=> + def <=>(other) + raise end end + p a.sort { |n,m| (n - m) * (2 ** 200) } + RUBY end it "compares values returned by block with 0" do diff --git a/spec/ruby/core/proc/fixtures/refined.rb b/spec/ruby/core/proc/fixtures/refined.rb index 5084408365fb5d..014e8695fdd633 100644 --- a/spec/ruby/core/proc/fixtures/refined.rb +++ b/spec/ruby/core/proc/fixtures/refined.rb @@ -21,31 +21,4 @@ def quiet end end end - - # Refines operators and element access, including Hash#[] with a String - # key, so specs can check the specialized call paths implementations use - # for them. - module Operators - refine Integer do - def +(other) - "plus(#{self},#{other})" - end - - def <(other) - "lt" - end - end - - refine Array do - def [](i) - "at#{i}" - end - end - - refine Hash do - def [](k) - "aref(#{k})" - end - end - end end diff --git a/spec/ruby/core/proc/fixtures/refined_basic_operations.rb b/spec/ruby/core/proc/fixtures/refined_basic_operations.rb new file mode 100644 index 00000000000000..d12dbab3a01d07 --- /dev/null +++ b/spec/ruby/core/proc/fixtures/refined_basic_operations.rb @@ -0,0 +1,33 @@ +# Refines basic operations like operators and element access, +# including Hash#[] with a String key, so specs can check +# the specialized call paths implementations use for them. +# Do this in a subprocess to not disable optimizations globally for the main process. +module Operators + refine Integer do + def +(other) + "plus(#{self},#{other})" + end + + def <(other) + "lt" + end + end + + refine Array do + def [](i) + "at#{i}" + end + end + + refine Hash do + def [](k) + "aref(#{k})" + end + end +end + +refined = -> a, b { [a + b, a < b] }.refined(Operators) +puts refined.call(1, 2) +puts -> a { a[0] }.refined(Operators).call([9]) +puts -> h { h["x"] }.refined(Operators).call({ "x" => 1 }) +puts -> a, b { a + b }.call(1, 2) diff --git a/spec/ruby/core/proc/refined_spec.rb b/spec/ruby/core/proc/refined_spec.rb index 36b787b28b1c2e..5656be5382aebf 100644 --- a/spec/ruby/core/proc/refined_spec.rb +++ b/spec/ruby/core/proc/refined_spec.rb @@ -110,11 +110,14 @@ def shout_hi end it "applies the refinements to operators and element access" do - refined = -> a, b { [a + b, a < b] }.refined(ProcRefinedSpecs::Operators) - refined.call(1, 2).should == ["plus(1,2)", "lt"] - -> a { a[0] }.refined(ProcRefinedSpecs::Operators).call([9]).should == "at0" - -> h { h["x"] }.refined(ProcRefinedSpecs::Operators).call({ "x" => 1 }).should == "aref(x)" - -> a, b { a + b }.call(1, 2).should == 3 + file = fixture(__FILE__, "refined_basic_operations.rb") + ruby_exe(file).should == <<~EXPECTED + plus(1,2) + lt + at0 + aref(x) + 3 + EXPECTED end it "keeps the refinements active when called via instance_eval, instance_exec and class_eval" do diff --git a/spec/ruby/shared/file/setgid.rb b/spec/ruby/shared/file/setgid.rb index 3b32ef5454475b..801d6f711579d0 100644 --- a/spec/ruby/shared/file/setgid.rb +++ b/spec/ruby/shared/file/setgid.rb @@ -1,5 +1,6 @@ describe :file_setgid, shared: true do - platform_is :darwin do + # Fails on RubyCI + quarantine! do # platform_is :darwin do it "accepts a path in a non-UTF-8, ASCII-compatible encoding containing non-ASCII characters" do utf8_path = tmp("file_predicate_utf8_path_\u{3042}.txt") # Can fail with UndefinedConversionError if tmp path has non-Shift_JIS chars (e.g. Emojis, Hangul, Cyrillic, accented letters) diff --git a/string.c b/string.c index c33c6589d75e70..5481705cf8b3b4 100644 --- a/string.c +++ b/string.c @@ -278,6 +278,12 @@ STR_EMBEDDABLE_P(long len, long termlen) return rb_gc_size_allocatable_p(rb_str_embed_size(len, termlen)); } +/* Substrings and duplicated strings that need a slot larger than this are shared + * instead of copied. Larger slots hold fewer objects per page and trigger GC + * more often, which outweighs the copy they save; see [Feature #22186] for the + * benchmarks. */ +#define STR_COPY_MAX_EMBED_SIZE 256 + static VALUE str_replace_shared_without_enc(VALUE str2, VALUE str); static VALUE str_new_frozen(VALUE klass, VALUE orig); static VALUE str_new_frozen_buffer(VALUE klass, VALUE orig, int copy_encoding); @@ -1977,16 +1983,11 @@ str_duplicate_setup_heap(VALUE klass, VALUE str, VALUE dup) str_duplicate_setup_encoding(str, dup, flags); } -/* Force duplicated strings above 256 bytes to be views rather than copies since - * copying will use memory and have significant overhead. - * Calculated as: 256 - header size - NUL terminator size */ -#define STR_DUPLICATE_MAX_EMBED_LEN ((long)(256 - offsetof(struct RString, as.embed) - 1)) - static inline VALUE str_duplicate(VALUE klass, VALUE str) { VALUE dup; - if (STR_EMBED_P(str) && RSTRING_LEN(str) <= STR_DUPLICATE_MAX_EMBED_LEN) { + if (STR_EMBED_P(str) && rb_str_embed_size(RSTRING_LEN(str), 1) <= STR_COPY_MAX_EMBED_SIZE) { dup = str_alloc_embed(klass, RSTRING_LEN(str) + TERM_LEN(str)); str_duplicate_setup_embed(klass, str, dup); @@ -2070,11 +2071,8 @@ rb_zjit_str_resurrect_fastpath(VALUE str, bool chilled, size_t *size_out, flags |= T_STRING; if (chilled) flags |= STR_CHILLED; - shape_id_t shape_id = rb_shape_transition_slot_size(ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, - rb_gc_size_slot_size(size)); - *size_out = size; - *flags_out = flags | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); + *flags_out = flags; *len_out = len; *byte_size_out = (size_t)(len + termlen); return true; @@ -3174,11 +3172,6 @@ rb_str_sublen(VALUE str, long pos) } } -/* Substrings that need a slot larger than this are shared instead of copied. - * Larger slots hold fewer objects per page and trigger GC more often, which - * outweighs the copy they save; see [Feature #22186] for the benchmarks. */ -#define STR_SUBSEQ_MAX_EMBED_SIZE 256 - static VALUE str_subseq(VALUE str, long beg, long len) { @@ -3203,7 +3196,7 @@ str_subseq(VALUE str, long beg, long len) const bool root_available = STR_SHARED_P(str) || RB_FL_TEST_RAW(str, FL_FREEZE | STR_CHILLED) == FL_FREEZE; const size_t max_embed_size = root_available ? - rb_gc_size_slot_size(sizeof(struct RString)) : STR_SUBSEQ_MAX_EMBED_SIZE; + rb_gc_size_slot_size(sizeof(struct RString)) : STR_COPY_MAX_EMBED_SIZE; const size_t embed_size = rb_str_embed_size(len, termlen); if (embed_size <= max_embed_size && rb_gc_size_allocatable_p(embed_size)) { diff --git a/test/-ext-/string/test_rb_str_dup.rb b/test/-ext-/string/test_rb_str_dup.rb index 3a39e0bf4e06d6..638fc7260c9fb8 100644 --- a/test/-ext-/string/test_rb_str_dup.rb +++ b/test/-ext-/string/test_rb_str_dup.rb @@ -2,17 +2,17 @@ require '-test-/string' class Test_RbStrDup < Test::Unit::TestCase - STR_DUPLICATE_MAX_EMBED_LEN = 256 - (RbConfig::SIZEOF["void*"] * 3) - 1 # From macro defined in string.c + STR_COPY_MAX_EMBED_SIZE = 256 - (RbConfig::SIZEOF["void*"] * 3) - 1 # From macro defined in string.c def test_nested_shared_non_frozen - orig_str = "a" * (STR_DUPLICATE_MAX_EMBED_LEN + 1) + orig_str = "a" * (STR_COPY_MAX_EMBED_SIZE + 1) str = Bug::String.rb_str_dup(Bug::String.rb_str_dup(orig_str)) assert_send([Bug::String, :shared_string?, str]) assert_not_send([Bug::String, :sharing_with_shared?, str], '[Bug #15792]') end def test_nested_shared_frozen - orig_str = "a" * (STR_DUPLICATE_MAX_EMBED_LEN + 1) + orig_str = "a" * (STR_COPY_MAX_EMBED_SIZE + 1) str = Bug::String.rb_str_dup(Bug::String.rb_str_dup(orig_str).freeze) assert_send([Bug::String, :shared_string?, str]) assert_not_send([Bug::String, :sharing_with_shared?, str], '[Bug #15792]') diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index cfb38e69750023..2378aaf4016b55 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -28,10 +28,10 @@ def test_memsize_of ObjectSpace.memsize_of(//.match(""))) end - STR_DUPLICATE_MAX_EMBED_LEN = 256 - (RbConfig::SIZEOF["void*"] * 3) - 1 # From macro defined in string.c + STR_COPY_MAX_EMBED_SIZE = 256 - (RbConfig::SIZEOF["void*"] * 3) - 1 # From macro defined in string.c def test_memsize_of_root_shared_string - a = "a" * (STR_DUPLICATE_MAX_EMBED_LEN + 1) + a = "a" * (STR_COPY_MAX_EMBED_SIZE + 1) b = a.dup c = nil ObjectSpace.each_object(String) {|x| break c = x if a == x and x.frozen?} diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 93187fe1ca99b6..5dfff01ec8cd79 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3644,7 +3644,7 @@ def test_substring_embed require 'objspace' - # 128 and 320 sit either side of STR_SUBSEQ_MAX_EMBED_SIZE in string.c, which + # 128 and 320 sit either side of STR_COPY_MAX_EMBED_SIZE in string.c, which # the copy has to fit in along with the header and the terminator substr = str.byteslice(320, 128) assert_equal "a" * 128, substr diff --git a/vm_trace.c b/vm_trace.c index 78ba3e24fd5996..16ff2417e11b97 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -150,7 +150,8 @@ update_global_event_hooks(rb_hook_list_t *list, rb_event_flag_t prev_events, rb_ // as for all ractors. That's not how it works right now, so we shouldn't rely on it apart from the // internal events. Since it doesn't work like this, we have to track more state with `ruby_vm_iseq_events_enabled`, // `ruby_vm_c_events_enabled`, etc. - rb_event_flag_t new_events_global = (ruby_vm_event_flags & ~prev_events) | new_events; + rb_event_flag_t prev_events_global = ruby_vm_event_flags; + rb_event_flag_t new_events_global = (prev_events_global & ~prev_events) | new_events; ruby_vm_event_flags = new_events_global; // Modify ISEQs or CCs to enable tracing @@ -179,6 +180,13 @@ update_global_event_hooks(rb_hook_list_t *list, rb_event_flag_t prev_events, rb_ rb_objspace_set_event_hook(new_events_global); } + // ZJIT's inline allocation fast path bypasses rb_newobj, so it can't fire the + // NEWOBJ internal event. Enabling such a hook invalidates the fast path code so + // allocation falls back to the interpreter, which fires the event. + if ((new_events_global & RUBY_INTERNAL_EVENT_NEWOBJ) && !(prev_events_global & RUBY_INTERNAL_EVENT_NEWOBJ)) { + rb_zjit_invalidate_newobj_hook(); + } + // Invalidate JIT code as needed if (new_iseq_events_p || clear_attr_ccs_p) { // Invalidate all code when ISEQs are modified to use trace_* insns above. diff --git a/zjit.c b/zjit.c index 4e3654b62cf972..52b80f536ce289 100644 --- a/zjit.c +++ b/zjit.c @@ -24,6 +24,7 @@ #include "ruby/debug.h" #include "internal/cont.h" #include "ractor_core.h" +#include "shape.h" // This build config impacts the pointer tagging scheme and we only want to // support one scheme for simplicity. @@ -35,6 +36,23 @@ enum zjit_struct_offsets { RUBY_OFFSET_THREAD_RACTOR = offsetof(rb_thread_t, ractor), }; +// Struct offsets that cannot be constants in the checked-in bindgen output +// (zjit/src/cruby_bindings.inc.rs) because they vary with the build target +// and configuration. For example, offsetof(rb_ractor_t, newobj_cache) depends +// on the sizes of pthread types embedded in rb_ractor_t, which differ across +// architectures and OSes, as well as on VM_CHECK_MODE and RACTOR_CHECK_MODE. +// This table is filled out at C compile time and read by Rust at JIT compile +// time. Offsets that are identical on all supported builds should be added to +// enum zjit_struct_offsets above instead. +struct rb_zjit_runtime_offsets { + int32_t ractor_newobj_cache; + int32_t ractor_objspace; +}; +const struct rb_zjit_runtime_offsets rb_zjit_runtime_offsets = { + .ractor_newobj_cache = offsetof(rb_ractor_t, newobj_cache), + .ractor_objspace = offsetof(rb_ractor_t, objspace), +}; + // Special JITFrame used by all C method calls. We don't control the native // stack layout for C frames, so cfp->jit_return points at this static frame // via the ZJIT_JIT_RETURN_C_FRAME sentinel instead of a per-call allocation. @@ -168,20 +186,35 @@ rb_zjit_singleton_class_p(VALUE klass) return RCLASS_SINGLETON_P(klass); } -/* - * These offsets differ between x68_64 and arm64, so we must generate them each - * time. We can't bake them into zjit_struct_offsets +/* Sets all of the required shape flags for the object including the layout type, + * the frozen status, and the slot size. Mimics `rb_newobj`. */ -size_t -rb_zjit_offset_ractor_newobj_cache(void) +VALUE +rb_zjit_new_obj_shape(VALUE flags, size_t alloc_size) { - return offsetof(rb_ractor_t, newobj_cache); -} + shape_id_t shape_id; + switch (flags & T_MASK) { + case T_OBJECT: + shape_id = ROOT_SHAPE_ID; + break; + case T_STRUCT: + shape_id = ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_EXTENDED; + break; + case T_DATA: + shape_id = ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA; + break; + default: + shape_id = ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER; + break; + } -size_t -rb_zjit_offset_ractor_objspace(void) -{ - return offsetof(rb_ractor_t, objspace); + if (flags & FL_FREEZE) { + shape_id = rb_shape_transition_frozen(shape_id); + } + + shape_id = rb_shape_transition_slot_size(shape_id, rb_gc_size_slot_size(alloc_size)); + + return (flags & SHAPE_FLAG_MASK) | ((VALUE)shape_id << SHAPE_FLAG_SHIFT); } VALUE diff --git a/zjit.h b/zjit.h index cabef7ef2b81b5..80259fd5dbcd19 100644 --- a/zjit.h +++ b/zjit.h @@ -88,16 +88,20 @@ void rb_zjit_mark_all_executable(void); void rb_zjit_iseq_free(const rb_iseq_t *iseq); void rb_zjit_invalidate_single_ractor(void); void rb_zjit_tracing_invalidate_all(void); +void rb_zjit_invalidate_newobj_hook(void); void rb_zjit_invalidate_no_singleton_class(VALUE klass); void rb_zjit_invalidate_root_box(void); void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); void rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp); void rb_zjit_materialize_frames_for_longjmp(const rb_execution_context_t *ec, rb_control_frame_t *cfp); size_t rb_zjit_hash_new_size(VALUE *flags_out); -bool rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, shape_id_t *shape_id_out); +VALUE rb_zjit_new_obj_shape(VALUE flags, size_t alloc_size); +bool rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, VALUE *flags_out); bool rb_zjit_str_resurrect_fastpath(VALUE str, bool chilled, size_t *size_out, VALUE *flags_out, long *len_out, size_t *byte_size_out); bool rb_zjit_array_dup_can_fastpath(VALUE ary, size_t *alloc_size_out, VALUE *flags_out, long *len_out); void rb_zjit_range_new_fastpath(bool exclude_end, size_t *alloc_size_out, VALUE *flags_out); +void rb_zjit_array_new_fastpath(size_t *alloc_size_out, VALUE *flags_out); +bool rb_zjit_newobj_hook_enabled_p(void); // Special value for cfp->jit_return that means "this is a C method frame, use // rb_zjit_c_frame as the JITFrame". We don't control the native stack layout @@ -135,6 +139,7 @@ static inline void rb_zjit_invalidate_no_ep_escape(const rb_iseq_t *iseq) {} static inline void rb_zjit_constant_state_changed(ID id) {} static inline void rb_zjit_invalidate_single_ractor(void) {} static inline void rb_zjit_tracing_invalidate_all(void) {} +static inline void rb_zjit_invalidate_newobj_hook(void) {} static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {} static inline void rb_zjit_invalidate_root_box(void) {} static inline void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame) {} diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index bcaff445c9b594..83a317213e9864 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -118,6 +118,7 @@ fn main() { .allowlist_function("rb_zjit_str_resurrect_fastpath") .allowlist_function("rb_zjit_array_dup_can_fastpath") .allowlist_function("rb_zjit_range_new_fastpath") + .allowlist_function("rb_zjit_array_new_fastpath") // For crashing .allowlist_function("rb_bug") @@ -172,6 +173,7 @@ fn main() { .allowlist_function("rb_gc_writebarrier") .allowlist_function("rb_gc_writebarrier_remember") .allowlist_function("rb_gc_register_mark_object") + .allowlist_function("rb_zjit_new_obj_shape") // VALUE variables for Ruby class objects .allowlist_var("rb_cBasicObject") @@ -338,6 +340,7 @@ fn main() { .allowlist_function("rb_zjit_insn_leaf") .allowlist_type("jit_bindgen_constants") .allowlist_type("zjit_struct_offsets") + .allowlist_var("rb_zjit_runtime_offsets") .allowlist_var("ZJIT_STACK_MAP_SHIFT") .allowlist_var("ZJIT_STACK_MAP_VREG_TAG") .allowlist_var("ZJIT_STACK_MAP_SKIP_TAG") diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 33816ef398c0f0..df6bcfbde5012c 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -13,7 +13,7 @@ use crate::backend::current::ALLOC_REGS; use crate::invariants::{ track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption, - track_root_box_assumption + track_root_box_assumption, track_no_newobj_hook_assumption }; use crate::gc::append_gc_offsets; use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; @@ -618,13 +618,13 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio gen_const_uint32(val.0) } Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), - Insn::NewArray { elements, state } => gen_new_array(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewArray { elements, state } => gen_new_array(jit, asm, function, opnds!(elements), &function.frame_state(*state)), Insn::NewHash { elements, state } => { let sym_keys = elements.iter().step_by(2).all(|&key| function.type_of(key).is_subtype(types::Symbol)); gen_new_hash(jit, asm, function, opnds!(elements), sym_keys, &function.frame_state(*state)) } Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, function, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), - Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(jit, asm, function, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(jit, asm, function, *val, opnd!(val), &function.frame_state(*state)), Insn::AdjustBounds { index, length } => gen_adjust_bounds(asm, opnd!(index), opnd!(length)), Insn::ArrayAref { array, index, .. } => gen_array_aref(asm, opnd!(array), opnd!(index)), @@ -634,7 +634,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, function, opnd!(val), &function.frame_state(*state)), - &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(jit, asm, class, &function.frame_state(state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(jit, asm, function, class, &function.frame_state(state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(jit, asm, function, *val, opnd!(val), *chilled, &function.frame_state(*state)), Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, function, opnds!(strings), &function.frame_state(*state)), &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), @@ -1006,6 +1006,9 @@ pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invari Invariant::NoTracePoint => { track_no_trace_point_assumption(code_ptr, side_exit_ptr, version); } + Invariant::NoNewObjHook => { + track_no_newobj_hook_assumption(code_ptr, side_exit_ptr, version); + } Invariant::NoEPEscape(iseq) => { track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, version); } @@ -2086,7 +2089,7 @@ fn gen_string_copy(jit: &mut JITState, asm: &mut Assembler, function: &Function, // pool). Here we choose an arbitrary threshold (128 bytes, or 16 stores), // above which we'll emit a C call to memcpy instead of multiple stores. if byte_size > STR_INLINE_STORE_MAX_BYTES { - return gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, full_flags, klass, + return gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, full_flags, klass, |asm, obj| { asm.store(Opnd::mem(VALUE_BITS, obj, RUBY_OFFSET_RSTRING_LEN), Opnd::Imm(len)); let src_obj = asm.load(Opnd::Value(src)); @@ -2108,7 +2111,7 @@ fn gen_string_copy(jit: &mut JITState, asm: &mut Assembler, function: &Function, let mut string_bytes = vec![0u8; padded_size]; string_bytes[..src_bytes.len()].copy_from_slice(src_bytes); - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, full_flags, klass, + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, full_flags, klass, |asm, obj| { asm.store(Opnd::mem(VALUE_BITS, obj, RUBY_OFFSET_RSTRING_LEN), Opnd::Imm(len)); for (i, chunk) in string_bytes.chunks_exact(8).enumerate() { @@ -2147,7 +2150,7 @@ fn gen_array_dup( let mut len: std::os::raw::c_long = 0; if unsafe { rb_zjit_array_dup_can_fastpath(src, &mut alloc_size, &mut flags, &mut len) } { let klass = unsafe { rb_cArray }; - return gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags.as_u64(), klass, |asm, obj| { + return gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.into(), klass, |asm, obj| { for i in 0..len { let elem = unsafe { rb_ary_entry(src, i) }; let offset = RUBY_OFFSET_RARRAY_AS_ARY + (i as i32) * SIZEOF_VALUE_I32; @@ -2169,6 +2172,7 @@ fn gen_array_dup( fn gen_new_array( jit: &mut JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { @@ -2181,12 +2185,12 @@ fn gen_new_array( return asm_ccall!(asm, rb_ec_ary_new_from_values, EC, num.into(), argv); } - let alloc_size = std::mem::size_of::(); - - let flags = (RUBY_T_ARRAY as u64) | (RARRAY_EMBED_FLAG as u64); + let mut alloc_size: usize = 0; + let mut flags: VALUE = VALUE(0); + unsafe { rb_zjit_array_new_fastpath(&mut alloc_size, &mut flags) }; let klass = unsafe { rb_cArray }; - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags, klass, |_asm, _obj| {}, |asm| { + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.into(), klass, |_asm, _obj| {}, |asm| { asm_ccall!(asm, rb_ec_ary_new_from_values, EC, 0i64.into(), Opnd::UImm(0)) }) } @@ -2479,7 +2483,7 @@ fn gen_new_hash( let alloc_size = unsafe { rb_zjit_hash_new_size(&mut flags) }; let klass = unsafe { rb_cHash }; - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags.into(), klass, + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.into(), klass, |asm, hash| { asm.store(Opnd::mem(VALUE_BITS, hash, RUBY_OFFSET_RHASH_IFNONE), Qnil.into()); }, @@ -2498,7 +2502,7 @@ fn gen_new_hash( let alloc_size = unsafe { rb_zjit_hash_new_size(&mut flags) }; let klass = unsafe { rb_cHash }; - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags.into(), klass, + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.into(), klass, |asm, hash| { asm.store(Opnd::mem(VALUE_BITS, hash, RUBY_OFFSET_RHASH_IFNONE), Qnil.into()); }, @@ -2556,7 +2560,7 @@ fn gen_new_range( asm.set_current_block(fast_block); let label = jit.get_label(asm, fast_block, hir_block_id); asm.write_label(label); - let range = gen_new_range_fixnum(jit, asm, low, high, flag, state); + let range = gen_new_range_fixnum(jit, asm, function, low, high, flag, state); asm.jmp(result_edge(range)); asm.set_current_block(slow_block); @@ -2578,6 +2582,7 @@ fn gen_new_range( fn gen_new_range_fixnum( jit: &mut JITState, asm: &mut Assembler, + function: &Function, low: lir::Opnd, high: lir::Opnd, flag: RangeType, @@ -2591,7 +2596,7 @@ fn gen_new_range_fixnum( }; let klass = unsafe { rb_cRange }; - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags.as_u64(), klass, + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.into(), klass, |asm, range| { asm.store(Opnd::mem(VALUE_BITS, range, RUBY_OFFSET_RSTRUCT_FIELDS_OBJ), Opnd::UImm(0)); asm.store(Opnd::mem(VALUE_BITS, range, RUBY_OFFSET_RSTRUCT_AS_ARY), low); @@ -2610,19 +2615,18 @@ fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, function: &Function, va asm_ccall!(asm, rb_obj_alloc, val) } -fn gen_object_alloc_class(jit: &mut JITState, asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { +fn gen_object_alloc_class(jit: &mut JITState, asm: &mut Assembler, function: &Function, class: VALUE, state: &FrameState) -> lir::Opnd { // Allocating an object for a known class with default allocator is leaf; see doc for // `ObjectAllocClass`. gen_prepare_leaf_call_with_gc(asm, state); if unsafe { rb_zjit_class_has_default_allocator(class) } { let mut alloc_size: usize = 0; - let mut shape_id: shape_id_t = 0; + let mut flags = VALUE(0); let has_fastpath = unsafe { - rb_zjit_class_allocate_instance_fastpath(class, &mut alloc_size, &mut shape_id) + rb_zjit_class_allocate_instance_fastpath(class, &mut alloc_size, &mut flags) }; if has_fastpath { - let flags = (RUBY_T_OBJECT as u64) | ((shape_id as u64) << RB_SHAPE_FLAG_SHIFT as u64); - gc_fastpath::gc_fastpath_new_obj(jit, asm, alloc_size, flags, class, |_asm, _obj| {}, |asm| { + gc_fastpath::gc_fastpath_new_obj(jit, asm, function, state, alloc_size, flags.as_u64(), class, |_asm, _obj| {}, |asm| { asm_ccall!(asm, rb_class_allocate_instance, class.into()) }) } else { diff --git a/zjit/src/codegen/gc_fastpath.rs b/zjit/src/codegen/gc_fastpath.rs index ff9532b0abd32a..c9d4be35165749 100644 --- a/zjit/src/codegen/gc_fastpath.rs +++ b/zjit/src/codegen/gc_fastpath.rs @@ -1,48 +1,24 @@ -use std::ffi::c_void; - use crate::backend::lir::{self, Assembler, EC, Opnd, Target, asm_comment}; use crate::cruby::{ RB_GC_ZJIT_FASTPATH_DEFAULT, RB_GC_ZJIT_FASTPATH_MMTK, RUBY_OFFSET_EC_THREAD_PTR, RUBY_OFFSET_RBASIC_FLAGS, RUBY_OFFSET_RBASIC_KLASS, - RUBY_OFFSET_THREAD_RACTOR, VALUE, VALUE_BITS, rb_zjit_offset_ractor_newobj_cache, - rb_zjit_offset_ractor_objspace, + RUBY_OFFSET_THREAD_RACTOR, VALUE, VALUE_BITS, rb_zjit_new_obj_shape, + rb_zjit_runtime_offsets, + rb_gc_zjit_default_new_obj_fastpath as RbGcZjitDefaultNewObjFastpath, + rb_gc_zjit_mmtk_new_obj_fastpath as RbGcZjitMmtkNewObjFastpath, }; -use super::JITState; +use crate::hir::{FrameState, Function, Invariant}; +use super::{JITState, gen_patch_point}; -#[repr(C)] -#[derive(Clone, Copy)] -struct RbGcZjitDefaultNewObjFastpath { - cursor_offset: usize, - cursor_end_offset: usize, - slot_size: usize, - total_allocated_objects_offset: usize, - flags: VALUE, - klass: VALUE, +impl Clone for RbGcZjitDefaultNewObjFastpath { + fn clone(&self) -> Self { *self } } +impl Copy for RbGcZjitDefaultNewObjFastpath {} -#[repr(C)] -#[derive(Clone, Copy)] -struct RbGcZjitMmtkNewObjFastpath { - objspace: *const c_void, - objspace_total_allocated_objects_offset: usize, - ractor_cache_mutator_offset: usize, - ractor_cache_bump_pointer_offset: usize, - ractor_cache_obj_free_parallel_buf_offset: usize, - ractor_cache_obj_free_parallel_count_offset: usize, - bump_pointer_cursor_offset: usize, - bump_pointer_limit_offset: usize, - min_obj_align: usize, - payload_size: usize, - total_alloc_size: usize, - allocation_semantics_default: u32, - gc_stress_p_func: usize, - newobj_tracing_p_func: usize, - post_alloc_func: usize, - obj_free_buf_capacity_minus_one: usize, - value_size_shift: usize, - flags: VALUE, - klass: VALUE, +impl Clone for RbGcZjitMmtkNewObjFastpath { + fn clone(&self) -> Self { *self } } +impl Copy for RbGcZjitMmtkNewObjFastpath {} #[repr(C)] union RbGcZjitFastpathData { @@ -63,6 +39,8 @@ unsafe extern "C" { klass: VALUE, fastpath: *mut RbGcZjitFastpath, ) -> bool; + + fn rb_zjit_newobj_hook_enabled_p() -> bool; } enum PreparedNewObjFastpath { @@ -73,16 +51,29 @@ enum PreparedNewObjFastpath { pub(super) fn gc_fastpath_new_obj( jit: &mut JITState, asm: &mut Assembler, + function: &Function, + state: &FrameState, alloc_size: usize, flags: u64, klass: VALUE, init: impl Fn(&mut Assembler, Opnd), slow_path: impl Fn(&mut Assembler) -> lir::Opnd, ) -> lir::Opnd { + let flags = unsafe { rb_zjit_new_obj_shape(VALUE(flags as usize), alloc_size) }.as_u64(); + let Some(fastpath) = prepare_new_obj_fastpath(alloc_size, flags, klass) else { return slow_path(asm); }; + // Both inline fast paths bump an allocation cursor without calling rb_newobj, + // so neither fires the NEWOBJ internal event. If such a hook is active, use the + // C path; otherwise assume none is active and install a patch point that + // discards this code if one is enabled later. + if unsafe { rb_zjit_newobj_hook_enabled_p() } { + return slow_path(asm); + } + gen_patch_point(jit, asm, function, &Invariant::NoNewObjHook, state); + asm_comment!(asm, "GC inline allocation"); let hir_block_id = asm.current_block().hir_block_id; @@ -130,7 +121,6 @@ fn prepare_new_obj_fastpath(alloc_size: usize, flags: u64, klass: VALUE) -> Opti let fastpath = unsafe { fastpath.data.mmtk }; if fastpath.objspace.is_null() || fastpath.gc_stress_p_func == 0 - || fastpath.newobj_tracing_p_func == 0 || fastpath.post_alloc_func == 0 || fastpath.min_obj_align == 0 || !fastpath.min_obj_align.is_power_of_two() @@ -184,9 +174,7 @@ fn emit_default_new_obj_fastpath( let thread = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); let ractor = asm.load(Opnd::mem(64, thread, RUBY_OFFSET_THREAD_RACTOR as i32)); - let ractor_objspace_offset: i32 = unsafe { rb_zjit_offset_ractor_objspace() } - .try_into() - .expect("ractor objspace offset fits in i32"); + let ractor_objspace_offset = unsafe { rb_zjit_runtime_offsets.ractor_objspace }; let gc_cache = asm.load(Opnd::mem(64, ractor, ractor_objspace_offset)); let cursor = asm.load(Opnd::mem(64, gc_cache, cursor_offset)); @@ -250,17 +238,11 @@ fn emit_mmtk_new_obj_fastpath( .try_into() .ok()?; let value_size_shift: u64 = fastpath.value_size_shift.try_into().ok()?; - let newobj_tracing_p_func = (fastpath.newobj_tracing_p_func != 0) - .then_some(fastpath.newobj_tracing_p_func as *const u8)?; let gc_stress_p_func = (fastpath.gc_stress_p_func != 0) .then_some(fastpath.gc_stress_p_func as *const u8)?; let post_alloc_func = (fastpath.post_alloc_func != 0) .then_some(fastpath.post_alloc_func as *const u8)?; - let event_hook = asm.ccall(newobj_tracing_p_func, vec![]); - asm.test(event_hook, event_hook); - asm.jnz(jit, miss.clone()); - let objspace_const = Opnd::const_ptr(fastpath.objspace); let gc_stress = asm.ccall(gc_stress_p_func, vec![objspace_const]); asm.test(gc_stress, gc_stress); @@ -269,9 +251,7 @@ fn emit_mmtk_new_obj_fastpath( let objspace = asm.load(objspace_const); let thread = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); let ractor = asm.load(Opnd::mem(64, thread, RUBY_OFFSET_THREAD_RACTOR as i32)); - let ractor_newobj_cache_offset: i32 = unsafe { rb_zjit_offset_ractor_newobj_cache() } - .try_into() - .expect("ractor newobj cache offset fits in i32"); + let ractor_newobj_cache_offset = unsafe { rb_zjit_runtime_offsets.ractor_newobj_cache }; let ractor_cache = asm.load(Opnd::mem(64, ractor, ractor_newobj_cache_offset)); let bump_pointer = asm.load(Opnd::mem( diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index d5d7d04d2ee11c..ee026aa304029f 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -3075,6 +3075,60 @@ fn test_fixnum_mod_negative() { assert_snapshot!(assert_compiles("[test(-7, 3), test(7, -3), test(-7, -3)]"), @"[2, -2, -1]"); } +#[test] +fn test_fixnum_mod_pow2_constant() { + // Modulo by a positive power-of-two constant is strength-reduced to FixnumAnd + eval(" + def test(a) = a % 8 + test(13) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles("[test(13), test(8), test(0), test(-1), test(-8), test(4611686018427387903), test(-4611686018427387904)]"), @"[5, 0, 0, 7, 0, 7, 0]"); +} + +#[test] +fn test_fixnum_mod_one_constant() { + eval(" + def test(a) = a % 1 + test(13) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles("[test(13), test(-13)]"), @"[0, 0]"); +} + +#[test] +fn test_fixnum_mod_negative_pow2_constant() { + // Only positive power-of-two divisors are strength-reduced + eval(" + def test(a) = a % -8 + test(13) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles("[test(13), test(-13)]"), @"[-3, -5]"); +} + +#[test] +fn test_fixnum_div_pow2_constant() { + // Division by a positive power-of-two constant is strength-reduced to FixnumRShift + eval(" + def test(a) = a / 8 + test(13) # profile opt_div + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("[test(13), test(-13), test(0), test(-1), test(4611686018427387903), test(-4611686018427387904)]"), @"[1, -2, 0, -1, 576460752303423487, -576460752303423488]"); +} + +#[test] +fn test_fixnum_div_negative_pow2_constant() { + // Only positive power-of-two divisors are strength-reduced + eval(" + def test(a) = a / -8 + test(13) # profile opt_div + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("[test(13), test(-13)]"), @"[-2, 1]"); +} + #[test] fn test_fixnum_aref_constant_index() { eval(" diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 969ceb39b9935e..05c070c06c7de7 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -122,9 +122,6 @@ unsafe extern "C" { ci: *const rb_callinfo, ) -> *const rb_callable_method_entry_t; - pub fn rb_zjit_offset_ractor_newobj_cache() -> usize; - pub fn rb_zjit_offset_ractor_objspace() -> usize; - // Floats within range will be encoded without creating objects in the heap. // (Range is 0x3000000000000001 to 0x4fffffffffffffff (1.7272337110188893E-77 to 2.3158417847463237E+77). pub fn rb_float_new(d: f64) -> VALUE; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 7a65e57403c754..c3b98679395b20 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1991,6 +1991,12 @@ pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; pub const ISEQ_BODY_OFFSET_OUTER_VARIABLES: zjit_struct_offsets = 288; pub const RUBY_OFFSET_THREAD_RACTOR: zjit_struct_offsets = 24; pub type zjit_struct_offsets = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_zjit_runtime_offsets { + pub ractor_newobj_cache: i32, + pub ractor_objspace: i32, +} pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; pub const RCLASS_OFFSET_PRIME_FIELDS_OBJ: jit_bindgen_constants = 40; @@ -2274,10 +2280,11 @@ unsafe extern "C" { pub fn rb_iseq_defined_string(type_: defined_type) -> VALUE; pub fn rb_zjit_profile_enable(iseq: *const rb_iseq_t); pub fn rb_zjit_hash_new_size(flags_out: *mut VALUE) -> usize; + pub fn rb_zjit_new_obj_shape(flags: VALUE, alloc_size: usize) -> VALUE; pub fn rb_zjit_class_allocate_instance_fastpath( klass: VALUE, size_out: *mut usize, - shape_id_out: *mut shape_id_t, + flags_out: *mut VALUE, ) -> bool; pub fn rb_zjit_str_resurrect_fastpath( str_: VALUE, @@ -2298,6 +2305,7 @@ unsafe extern "C" { alloc_size_out: *mut usize, flags_out: *mut VALUE, ); + pub fn rb_zjit_array_new_fastpath(alloc_size_out: *mut usize, flags_out: *mut VALUE); pub fn rb_profile_frames( start: ::std::os::raw::c_int, limit: ::std::os::raw::c_int, @@ -2308,6 +2316,7 @@ unsafe extern "C" { pub fn rb_profile_frame_absolute_path(frame: VALUE) -> VALUE; pub fn rb_profile_frame_full_label(frame: VALUE) -> VALUE; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub static rb_zjit_runtime_offsets: rb_zjit_runtime_offsets; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_zjit_insn_to_bare_insn(insn: ::std::os::raw::c_int) -> ::std::os::raw::c_int; pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 56ac49e6f6fad3..b883b999014153 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -169,6 +169,10 @@ pub enum Invariant { }, /// TracePoint is not enabled. If TracePoint is enabled, this is invalidated. NoTracePoint, + /// No NEWOBJ internal event hook is active. The inline allocation fast path + /// bypasses rb_newobj, so it can't fire NEWOBJ; this is invalidated when such + /// a hook is enabled. + NoNewObjHook, /// cfp->ep is not escaped to the heap on the ISEQ NoEPEscape(IseqPtr), /// There is one ractor running. If a non-root ractor gets spawned, this is invalidated. @@ -312,6 +316,7 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> { write!(f, ")") } Invariant::NoTracePoint => write!(f, "NoTracePoint"), + Invariant::NoNewObjHook => write!(f, "NoNewObjHook"), Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)), Invariant::SingleRactorMode => write!(f, "SingleRactorMode"), Invariant::NoSingletonClass { klass } => { @@ -6005,6 +6010,9 @@ impl Function { /// /// It can fold fixnum math, truthiness tests, and branches with constant conditionals. fn fold_constants(&mut self) { + fn is_power_of_two(d: i64) -> bool { + d > 0 && (d & (d - 1)) == 0 + } // TODO(max): Determine if it's worth it for us to reflow types after each branch // simplification. This means that we can have nice cascading optimizations if what used to // be a union of two different basic block arguments now has a single value. @@ -6177,6 +6185,19 @@ impl Function { &Insn::FixnumDiv { left, right, .. } => { match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { (_, Some(1)) => { self.make_equal_to(insn_id, left); continue; } + // Strength-reduce division by a power of two to an arithmetic right + // shift. Both Ruby's Integer#/ and a sign-extending shift round the + // quotient towards negative infinity, so this holds for all fixnums. + (None, Some(d)) if is_power_of_two(d) => { + let shift = self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_isize(d.trailing_zeros() as isize)) }); + self.insn_types[shift.0] = self.infer_type(shift); + new_insns.push(shift); + let replacement = self.new_insn(Insn::FixnumRShift { left, right: shift }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + new_insns.push(replacement); + continue; + } _ => {} } self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { @@ -6191,6 +6212,22 @@ impl Function { }) } &Insn::FixnumMod { left, right, .. } => { + match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { + // Strength-reduce modulo by a power of two to a bitwise AND. The sign + // of Ruby's Integer#% follows the (positive) divisor, so the result is + // in [0, d), which matches two's complement AND for all fixnums. + (None, Some(d)) if is_power_of_two(d) => { + let mask = self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_isize((d - 1) as isize)) }); + self.insn_types[mask.0] = self.infer_type(mask); + new_insns.push(mask); + let replacement = self.new_insn(Insn::FixnumAnd { left, right: mask }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + new_insns.push(replacement); + continue; + } + _ => {} + } self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { (Some(l), Some(r)) if r != 0 => { let l_obj = VALUE::fixnum_from_isize(l as isize); diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 471a66d02d2867..6b18f2aaa7cc1f 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -542,6 +542,194 @@ mod hir_opt_tests { "); } + #[test] + fn test_reduce_fixnum_mod_pow2() { + eval(" + def test(n) + n % 8 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[8] = Const Value(8) + PatchPoint MethodRedefined(Integer@0x1008, %@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v28:Fixnum[7] = Const Value(7) + v29:Fixnum = FixnumAnd v26, v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_reduce_fixnum_mod_non_pow2() { + eval(" + def test(n) + n % 6 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Integer@0x1008, %@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Fixnum = FixnumMod v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_reduce_fixnum_mod_negative_pow2() { + eval(" + def test(n) + n % -8 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-8] = Const Value(-8) + PatchPoint MethodRedefined(Integer@0x1008, %@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Fixnum = FixnumMod v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_reduce_fixnum_div_pow2() { + eval(" + def test(n) + n / 8 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[8] = Const Value(8) + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v28:Fixnum[3] = Const Value(3) + v29:Fixnum = FixnumRShift v26, v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_reduce_fixnum_div_non_pow2() { + eval(" + def test(n) + n / 6 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Integer = FixnumDiv v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_reduce_fixnum_div_negative_pow2() { + eval(" + def test(n) + n / -8 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-8] = Const Value(-8) + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Integer = FixnumDiv v26, v15 + CheckInterrupts + Return v27 + "); + } + #[test] fn test_fold_fixnum_mod_zero_by_zero() { eval(" diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index 6a4ad3ff5a2687..b29e470f61c246 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -88,6 +88,9 @@ pub struct Invariants { /// Set of patch points that assume that the TracePoint is not enabled no_trace_point_patch_points: HashSet, + /// Set of patch points that assume no NEWOBJ internal event hook is active + no_newobj_hook_patch_points: HashSet, + /// Set of patch points that assume that the interpreter is running with only one ractor single_ractor_patch_points: HashSet, @@ -463,6 +466,42 @@ pub extern "C" fn rb_zjit_tracing_invalidate_all() { }); } +/// Track the JIT code that assumes no NEWOBJ internal event hook is active +pub fn track_no_newobj_hook_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_newobj_hook_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Callback for when a NEWOBJ internal event hook is enabled. The inline +/// allocation fast path bypasses rb_newobj, so it never fires NEWOBJ; invalidate +/// every block that assumed no such hook was active so it falls back to the +/// interpreter, which fires the event. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_newobj_hook() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().no_newobj_hook_patch_points); + + // Invalidate all patch points for the no NEWOBJ hook assumption + compile_patch_points!(cb, patch_points, NewObjHook, "NEWOBJ hook enabled, invalidating no NEWOBJ hook assumption"); + + cb.mark_all_executable(); + }); +} + /// Track the JIT code that assumes only the root box is active pub fn track_root_box_assumption( patch_point_ptr: CodePtr, diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 4323545bd9111b..f7c49a3931fd46 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -221,6 +221,7 @@ make_counters! { exit_patchpoint_method_redefined, exit_patchpoint_stable_constant_names, exit_patchpoint_no_tracepoint, + exit_patchpoint_no_newobj_hook, exit_patchpoint_no_ep_escape, exit_patchpoint_single_ractor_mode, exit_patchpoint_no_singleton_class, @@ -645,6 +646,8 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { => exit_patchpoint_stable_constant_names, PatchPoint(Invariant::NoTracePoint) => exit_patchpoint_no_tracepoint, + PatchPoint(Invariant::NoNewObjHook) + => exit_patchpoint_no_newobj_hook, PatchPoint(Invariant::NoEPEscape(_)) => exit_patchpoint_no_ep_escape, PatchPoint(Invariant::SingleRactorMode)