diff --git a/common.mk b/common.mk index a083265aac3441..8feb982f6bda55 100644 --- a/common.mk +++ b/common.mk @@ -1344,11 +1344,11 @@ $(BUILTIN_BINARY:no=builtin)_binary.rbbin: $(BUILTIN_RB_INCS): $(tooldir)/mk_builtin_loader.rb $(DUMP_AST_TARGET) -dump_ast$(BUILD_EXEEXT): $(tooldir)/dump_ast.c $(LIBPRISM_OBJS) revision.h +dump_ast$(BUILD_EXEEXT): $(tooldir)/dump_ast.c $(LIBPRISM_OBJS) $(ECHO) compiling $@ $(Q) $(CC) $(CFLAGS) $(OUTFLAG)$@ $(INCFLAGS) $(tooldir)/dump_ast.c $(LIBPRISM_OBJS) -build-tool/Makefile: $(tooldir)/dump_ast.mkmf.rb prism-srcs prism-incs revision.h +build-tool/Makefile: $(tooldir)/dump_ast.mkmf.rb prism-srcs prism-incs +$(BASERUBY) -s $(tooldir)/dump_ast.mkmf.rb \ "-INCFLAGS=$(INCFLAGS)" "-make=$(MAKE)" "-objext=$(OBJEXT)" \ build-tool $(tooldir)/dump_ast.c dump_ast.$(OBJEXT) $(LIBPRISM_OBJS) diff --git a/doc/jit/zjit.md b/doc/jit/zjit.md index fc69905749468b..e933afcc48055a 100644 --- a/doc/jit/zjit.md +++ b/doc/jit/zjit.md @@ -327,6 +327,13 @@ Collect stats without printing (access via `RubyVM::ZJIT.stats` in Ruby): ./miniruby --zjit-stats=quiet script.rb ``` +Dump stats to a file. Name the file with a `.json` extension to write the +stats as pretty-printed JSON instead of the human-readable text format: + +```bash +ruby --zjit-stats=stats.json script.rb +``` + ### Accessing Stats in Ruby ```ruby diff --git a/gc/default/default.c b/gc/default/default.c index 559598ed1ebe27..bc44e57fe7922b 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -502,6 +502,13 @@ typedef struct mark_stack { typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d); +typedef struct rb_heap_newobj { + uintptr_t alloc_cursor; + uintptr_t alloc_cursor_end; + struct free_region *alloc_next_region; + struct heap_page *alloc_using_page; +} rb_heap_newobj_t; + typedef struct rb_heap_struct { short slot_size; @@ -518,12 +525,7 @@ typedef struct rb_heap_struct { size_t empty_slots; /* Bump-pointer allocation state; only this objspace's owner thread writes it. */ - struct { - uintptr_t alloc_cursor; - uintptr_t alloc_cursor_end; - struct free_region *alloc_next_region; - struct heap_page *alloc_using_page; - } newobj; + rb_heap_newobj_t newobj; struct heap_page *free_pages; struct ccan_list_head pages; @@ -2981,10 +2983,40 @@ bool rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE flags, VALUE klass, struct rb_gc_zjit_fastpath *fastpath) { - /* Bump-pointer allocation state lives in the per-objspace heaps, but ZJIT's inline - * fastpath assumes a separate cache structure; report "no fastpath" (as gc/wbcheck - * does). The heaps are single-writer, so one could be offered later. */ +#if USE_ZJIT + size_t heap_idx = 0; + size_t slot_size = 0; + for (; heap_idx < HEAP_COUNT; heap_idx++) { + if (alloc_size + RVALUE_OVERHEAD <= pool_slot_sizes[heap_idx]) { + slot_size = pool_slot_sizes[heap_idx]; + break; + } + } + if (slot_size == 0) return false; + +#undef heaps + size_t base = offsetof(rb_objspace_t, heaps) + + heap_idx * sizeof(rb_heap_t) + + offsetof(rb_heap_t, newobj); +#define heaps objspace->heaps + + struct rb_gc_zjit_default_new_obj_fastpath default_fastpath = { + base + offsetof(rb_heap_newobj_t, alloc_cursor), + base + offsetof(rb_heap_newobj_t, alloc_cursor_end), + slot_size, + base - offsetof(rb_heap_t, newobj) + offsetof(rb_heap_t, total_allocated_objects), + flags, + klass + }; + + memset(fastpath, 0, sizeof(*fastpath)); + fastpath->kind = RB_GC_ZJIT_FASTPATH_DEFAULT; + memcpy(fastpath->data.words, &default_fastpath, sizeof(default_fastpath)); + + return true; +#else return false; +#endif } NOINLINE(static VALUE newobj_refill(rb_objspace_t *objspace, size_t heap_idx)); @@ -3071,6 +3103,11 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, int wb_protec obj = newobj_alloc(objspace, heap_idx); newobj_init(klass, flags, wb_protected, objspace, obj); + if (RB_UNLIKELY(ruby_gc_stressful)) { + rb_heap_t *heap = &heaps[heap_idx]; + heap->newobj.alloc_cursor_end = heap->newobj.alloc_cursor; + } + return obj; } @@ -5387,6 +5424,9 @@ rgengc_check_relation(rb_objspace_t *objspace, VALUE obj) if (objspace->rgengc.parent_object_old_p) { if (RVALUE_WB_UNPROTECTED(objspace, obj) || !RVALUE_OLD_P(objspace, obj)) { rgengc_remember(objspace, objspace->rgengc.parent_object); + /* It is in the rememberset now, so its remaining children have nothing left + * to ask for: stop testing them. */ + objspace->rgengc.parent_object_old_p = false; } } } diff --git a/gc/default/zjit_fastpath.h b/gc/default/zjit_fastpath.h index 3b72a89fc38a3f..4d5e45dcdc07a7 100644 --- a/gc/default/zjit_fastpath.h +++ b/gc/default/zjit_fastpath.h @@ -11,6 +11,7 @@ struct rb_gc_zjit_default_new_obj_fastpath { size_t cursor_offset; size_t cursor_end_offset; size_t slot_size; + size_t total_allocated_objects_offset; VALUE flags; VALUE klass; }; diff --git a/gc/gc.h b/gc/gc.h index d5f32df780ff37..9825dd06d4b00f 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -123,10 +123,10 @@ MODULAR_GC_FN void rb_gc_rp(VALUE); MODULAR_GC_FN void rb_gc_handle_weak_references(VALUE obj); MODULAR_GC_FN bool rb_gc_obj_needs_cleanup_p(VALUE obj); +void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context); #if USE_MODULAR_GC MODULAR_GC_FN bool rb_gc_event_hook_required_p(rb_event_flag_t event); MODULAR_GC_FN void *rb_gc_get_ractor_newobj_cache(void); -MODULAR_GC_FN void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context); MODULAR_GC_FN void rb_gc_move_obj_during_marking(VALUE from, VALUE to); MODULAR_GC_FN void rb_gc_print_backtrace(); #endif diff --git a/include/ruby/st.h b/include/ruby/st.h index 8ac34468844f39..bd808c3115e034 100644 --- a/include/ruby/st.h +++ b/include/ruby/st.h @@ -78,13 +78,16 @@ struct st_table_entry; /* defined in st.c */ struct st_table { /* Cached features of the table -- see st.c for more details. */ - unsigned char entry_power, bin_power, size_ind, entries_start; + unsigned char entry_power, bin_power, size_ind; /* How many times the table was rebuilt. */ - unsigned int rebuilds_num; + unsigned char rebuilds_num; + /* Start index of entries in array entries. */ + unsigned int entries_start; + const struct st_hash_type *type; /* Number of entries currently in the table. */ st_index_t num_entries; - /* Start and bound index of entries in array entries. + /* bound index of entries in array entries. entries_starts and entries_bound are in interval [0,allocated_entries]. */ st_index_t entries_bound; diff --git a/internal/set_table.h b/internal/set_table.h index 8c568d4fbf30c9..169b8e78c949ed 100644 --- a/internal/set_table.h +++ b/internal/set_table.h @@ -9,14 +9,17 @@ typedef struct set_table_entry set_table_entry; struct set_table { /* Cached features of the table -- see st.c for more details. */ - unsigned char entry_power, bin_power, size_ind, entries_start; + unsigned char entry_power, bin_power, size_ind; /* How many times the table was rebuilt. */ - unsigned int rebuilds_num; + unsigned char rebuilds_num; + + /* Start index of entries in array entries. */ + unsigned int entries_start; const struct st_hash_type *type; /* Number of entries currently in the table. */ st_index_t num_entries; - /* Start and bound index of entries in array entries. + /* bound index of entries in array entries. entries_starts and entries_bound are in interval [0,allocated_entries]. */ st_index_t entries_bound; diff --git a/st.c b/st.c index 3bedde661f2124..edc7e42f2eb552 100644 --- a/st.c +++ b/st.c @@ -131,7 +131,7 @@ #define ATTRIBUTE_UNUSED #endif -#define MAX_ENTRIES_START ((unsigned char)-1) +#define MAX_ENTRIES_START ((unsigned int)-1) /* The type of hashes. */ typedef st_index_t st_hash_t; @@ -1387,10 +1387,7 @@ update_range_for_deleted(st_table *tab, st_index_t n) st_index_t bound = tab->entries_bound; st_table_entry *entries = tab->entries; while (start < bound && DELETED_ENTRY_P(&entries[start])) start++; - if (start > MAX_ENTRIES_START) { - start = MAX_ENTRIES_START; - } - tab->entries_start = start; + tab->entries_start = start > MAX_ENTRIES_START ? MAX_ENTRIES_START : (unsigned int)start; } } @@ -3086,10 +3083,7 @@ set_update_range_for_deleted(set_table *tab, st_index_t n) st_index_t bound = tab->entries_bound; set_table_entry *entries = tab->entries; while (start < bound && DELETED_ENTRY_P(&entries[start])) start++; - if (start > MAX_ENTRIES_START) { - start = MAX_ENTRIES_START; - } - tab->entries_start = start; + tab->entries_start = start > MAX_ENTRIES_START ? MAX_ENTRIES_START : (unsigned int)start; } } diff --git a/string.c b/string.c index bc5434ef6ae1f9..c33c6589d75e70 100644 --- a/string.c +++ b/string.c @@ -3174,6 +3174,11 @@ rb_str_sublen(VALUE str, long pos) } } +/* Substrings that need a slot larger than this are shared instead of copied. + * Larger slots hold fewer objects per page and trigger GC more often, which + * outweighs the copy they save; see [Feature #22186] for the benchmarks. */ +#define STR_SUBSEQ_MAX_EMBED_SIZE 256 + static VALUE str_subseq(VALUE str, long beg, long len) { @@ -3193,12 +3198,19 @@ str_subseq(VALUE str, long beg, long len) return str2; } - str2 = str_alloc_heap(rb_cString); - if (str_embed_capa(str2) >= len + termlen) { + /* Sharing allocates a shared root as well unless str can be one itself, so + * a copy is worth a larger slot only when it saves that second object. */ + const bool root_available = STR_SHARED_P(str) || + RB_FL_TEST_RAW(str, FL_FREEZE | STR_CHILLED) == FL_FREEZE; + const size_t max_embed_size = root_available ? + rb_gc_size_slot_size(sizeof(struct RString)) : STR_SUBSEQ_MAX_EMBED_SIZE; + const size_t embed_size = rb_str_embed_size(len, termlen); + + if (embed_size <= max_embed_size && rb_gc_size_allocatable_p(embed_size)) { + str2 = str_alloc_embed(rb_cString, len + termlen); char *ptr2 = RSTRING(str2)->as.embed.ary; - STR_SET_EMBED(str2); memcpy(ptr2, RSTRING_PTR(str) + beg, len); - TERM_FILL(ptr2+len, termlen); + TERM_FILL(ptr2 + len, termlen); STR_SET_LEN(str2, len); if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) { @@ -3208,6 +3220,7 @@ str_subseq(VALUE str, long beg, long len) RB_GC_GUARD(str); } else { + str2 = str_alloc_heap(rb_cString); str_replace_shared(str2, str); RUBY_ASSERT(!STR_EMBED_P(str2)); if (ENC_CODERANGE(str) != ENC_CODERANGE_7BIT) { diff --git a/test/-ext-/string/test_rb_str_dup.rb b/test/-ext-/string/test_rb_str_dup.rb index 13f07e290613f8..3a39e0bf4e06d6 100644 --- a/test/-ext-/string/test_rb_str_dup.rb +++ b/test/-ext-/string/test_rb_str_dup.rb @@ -2,7 +2,7 @@ require '-test-/string' class Test_RbStrDup < Test::Unit::TestCase - STR_DUPLICATE_MAX_EMBED_LEN = 999 # From macro defined in string.c + STR_DUPLICATE_MAX_EMBED_LEN = 256 - (RbConfig::SIZEOF["void*"] * 3) - 1 # From macro defined in string.c def test_nested_shared_non_frozen orig_str = "a" * (STR_DUPLICATE_MAX_EMBED_LEN + 1) diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index d16ffcba2e19ee..93187fe1ca99b6 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3639,6 +3639,27 @@ def test_shared_middle_string_terminator refute_includes ObjectSpace.dump(substr), ' "shared":true,' end + def test_substring_embed + str = "a" * 448 + + require 'objspace' + + # 128 and 320 sit either side of STR_SUBSEQ_MAX_EMBED_SIZE in string.c, which + # the copy has to fit in along with the header and the terminator + substr = str.byteslice(320, 128) + assert_equal "a" * 128, substr + assert_includes ObjectSpace.dump(substr), ' "embedded":true,' + + substr = str.byteslice(128, 320) + assert_equal "a" * 320, substr + assert_includes ObjectSpace.dump(substr), ' "shared":true,' + + # A frozen source is a shared root itself, so the same substring is shared + substr = str.freeze.byteslice(320, 128) + assert_equal "a" * 128, substr + assert_includes ObjectSpace.dump(substr), ' "shared":true,' + end + def test_unknown_string_option str = nil assert_nothing_raised(SyntaxError) do diff --git a/test/ruby/test_zjit_cli.rb b/test/ruby/test_zjit_cli.rb index 877d345fb32f9c..7b94019067170e 100644 --- a/test/ruby/test_zjit_cli.rb +++ b/test/ruby/test_zjit_cli.rb @@ -82,6 +82,24 @@ def test = 42 assert_equal("true\n", out) assert_equal stats_header, File.open(stats_file) {|f| f.gets(chomp: true)}, "should be overwritten" } + + # With --zjit-stats= ending in .json, stats should be dumped as JSON + Tempfile.create(["zjit-stats-", ".json"]) {|tmp| + stats_file = tmp.path + tmp.puts("Lorem ipsum dolor sit amet, consectetur adipiscing elit, ...") + tmp.close + + out, err, status = eval_with_jit(script, stats: stats_file) + assert_success(out, err, status) + refute_includes(err, stats_header) + assert_equal("true\n", out) + + require "json" + json = JSON.parse(File.read(stats_file)) + assert_kind_of Hash, json, "should be JSON" + assert json.key?("compiled_iseq_count"), "should contain stats keys" + refute_includes File.read(stats_file), stats_header, "should not contain the text stats header" + } end def test_enable_through_env diff --git a/tool/dump_ast.c b/tool/dump_ast.c index 5e6038513eb606..0905dab3372734 100644 --- a/tool/dump_ast.c +++ b/tool/dump_ast.c @@ -2,7 +2,6 @@ #include #include #include -#include "revision.h" /* * When prism is compiled as part of CRuby, the xmalloc/xfree/etc. macros are @@ -28,25 +27,12 @@ print_error(const pm_diagnostic_t *diagnostic, void *data) fprintf(stderr, "%" PRIi32 ":%" PRIu32 ":%s\n", line_column.line, line_column.column, pm_diagnostic_message(diagnostic)); } -#if defined(RUBY_RELEASE_DATETIME) && defined(RUBY_RELEASE_DATETIME) -# define SHOW_PROGRAM_VERSION 2 -#elif defined(RUBY_RELEASE_DATETIME) || defined(RUBY_RELEASE_DATETIME) -# define SHOW_PROGRAM_VERSION 1 -#else -# define SHOW_PROGRAM_VERSION 0 -#endif -#if SHOW_PROGRAM_VERSION -# define usage_versions "and program versions" -#else -# define usage_versions "version" -#endif - static void usage(const char *prog) { fprintf(stderr, "Usage: %s [options]... \n" "Options:\n" - " -v, --version: show Prism " usage_versions "\n" + " -v, --version: show Prism version\n" " -h, --help: show this message\n" "", prog); } @@ -69,21 +55,7 @@ main(int argc, const char *argv[]) if (!arg[2]) break; if (strcmp(arg + 2, "version") == 0) { version: - fputs("Prism " PRISM_VERSION -#if SHOW_PROGRAM_VERSION - " [" -# ifdef RUBY_RELEASE_DATETIME - RUBY_RELEASE_DATETIME -# endif -# if SHOW_PROGRAM_VERSION > 1 - " " -# endif -# ifdef RUBY_REVISION - RUBY_REVISION -# endif - "]" -#endif - "\n", stdout); + fputs("Prism " PRISM_VERSION "\n", stdout); return EXIT_SUCCESS; } if (strcmp(arg + 2, "help") == 0) { diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 488f4f10aeabef..62a917ec8386ae 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -6141,6 +6141,10 @@ enum method_explorer_type { mexp_search_super, }; +ALWAYS_INLINE(static VALUE vm_sendish(struct rb_execution_context_struct *ec, + struct rb_control_frame_struct *reg_cfp, + struct rb_call_data *cd, VALUE block_handler, + enum method_explorer_type method_explorer)); static inline VALUE vm_sendish( struct rb_execution_context_struct *ec, diff --git a/zjit.c b/zjit.c index 2953a694fd3c83..4e3654b62cf972 100644 --- a/zjit.c +++ b/zjit.c @@ -168,12 +168,22 @@ rb_zjit_singleton_class_p(VALUE klass) return RCLASS_SINGLETON_P(klass); } +/* + * These offsets differ between x68_64 and arm64, so we must generate them each + * time. We can't bake them into zjit_struct_offsets + */ size_t rb_zjit_offset_ractor_newobj_cache(void) { return offsetof(rb_ractor_t, newobj_cache); } +size_t +rb_zjit_offset_ractor_objspace(void) +{ + return offsetof(rb_ractor_t, objspace); +} + VALUE rb_zjit_defined_ivar(VALUE obj, ID id, VALUE pushval) { diff --git a/zjit.rb b/zjit.rb index a579827bcd7062..dd5c3032dae575 100644 --- a/zjit.rb +++ b/zjit.rb @@ -285,7 +285,12 @@ def print_stats def print_stats_file filename = Primitive.rb_zjit_get_stats_file_path_p File.open(filename, "wb") do |file| - file.write stats_string + if filename.end_with?(".json") + require "json" + file.write(JSON.pretty_generate(stats)) + else + file.write stats_string + end end end diff --git a/zjit/src/codegen/gc_fastpath.rs b/zjit/src/codegen/gc_fastpath.rs index c72766c03b95a5..ff9532b0abd32a 100644 --- a/zjit/src/codegen/gc_fastpath.rs +++ b/zjit/src/codegen/gc_fastpath.rs @@ -5,6 +5,7 @@ use crate::cruby::{ RB_GC_ZJIT_FASTPATH_DEFAULT, RB_GC_ZJIT_FASTPATH_MMTK, RUBY_OFFSET_EC_THREAD_PTR, RUBY_OFFSET_RBASIC_FLAGS, RUBY_OFFSET_RBASIC_KLASS, RUBY_OFFSET_THREAD_RACTOR, VALUE, VALUE_BITS, rb_zjit_offset_ractor_newobj_cache, + rb_zjit_offset_ractor_objspace, }; use super::JITState; @@ -14,6 +15,7 @@ struct RbGcZjitDefaultNewObjFastpath { cursor_offset: usize, cursor_end_offset: usize, slot_size: usize, + total_allocated_objects_offset: usize, flags: VALUE, klass: VALUE, } @@ -178,13 +180,14 @@ fn emit_default_new_obj_fastpath( let cursor_offset: i32 = fastpath.cursor_offset.try_into().ok()?; let cursor_end_offset: i32 = fastpath.cursor_end_offset.try_into().ok()?; let slot_size: u64 = fastpath.slot_size.try_into().ok()?; + let total_allocated_objects_offset: i32 = fastpath.total_allocated_objects_offset.try_into().ok()?; let thread = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); let ractor = asm.load(Opnd::mem(64, thread, RUBY_OFFSET_THREAD_RACTOR as i32)); - let ractor_newobj_cache_offset: i32 = unsafe { rb_zjit_offset_ractor_newobj_cache() } + let ractor_objspace_offset: i32 = unsafe { rb_zjit_offset_ractor_objspace() } .try_into() - .expect("ractor newobj cache offset fits in i32"); - let gc_cache = asm.load(Opnd::mem(64, ractor, ractor_newobj_cache_offset)); + .expect("ractor objspace offset fits in i32"); + let gc_cache = asm.load(Opnd::mem(64, ractor, ractor_objspace_offset)); let cursor = asm.load(Opnd::mem(64, gc_cache, cursor_offset)); let cursor_end = asm.load(Opnd::mem(64, gc_cache, cursor_end_offset)); @@ -194,6 +197,9 @@ fn emit_default_new_obj_fastpath( asm.jl(jit, miss.clone()); asm.store(Opnd::mem(64, gc_cache, cursor_offset), new_cursor); + let total_allocated = asm.load(Opnd::mem(64, gc_cache, total_allocated_objects_offset)); + let new_total = asm.add(total_allocated, Opnd::UImm(1)); + asm.store(Opnd::mem(64, gc_cache, total_allocated_objects_offset), new_total); asm.store( Opnd::mem(VALUE_BITS, cursor, RUBY_OFFSET_RBASIC_FLAGS), fastpath.flags.as_u64().into(), diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 00265ae0d7df5c..969ceb39b9935e 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -123,6 +123,7 @@ unsafe extern "C" { ) -> *const rb_callable_method_entry_t; pub fn rb_zjit_offset_ractor_newobj_cache() -> usize; + pub fn rb_zjit_offset_ractor_objspace() -> usize; // Floats within range will be encoded without creating objects in the heap. // (Range is 0x3000000000000001 to 0x4fffffffffffffff (1.7272337110188893E-77 to 2.3158417847463237E+77). diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 3e84f0d4eb5a7f..7a65e57403c754 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2027,6 +2027,7 @@ pub struct rb_gc_zjit_default_new_obj_fastpath { pub cursor_offset: usize, pub cursor_end_offset: usize, pub slot_size: usize, + pub total_allocated_objects_offset: usize, pub flags: VALUE, pub klass: VALUE, } diff --git a/zjit/src/options.rs b/zjit/src/options.rs index c0cd8c4aabc7f3..dc696dcdb85cc8 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -238,7 +238,7 @@ pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ ("--zjit-stats-quiet", "Collect ZJIT stats and suppress output."), ("--zjit-stats[=file]", - "Collect ZJIT stats (=file to write to a file)."), + "Collect ZJIT stats (=file to write; .json for JSON)."), ("--zjit-disable", "Disable ZJIT for lazily enabling it with RubyVM::ZJIT.enable."), ("--zjit-perf[=iseq|hir]",