diff options
Diffstat (limited to 'yjit/src/stats.rs')
-rw-r--r-- | yjit/src/stats.rs | 1018 |
1 files changed, 1018 insertions, 0 deletions
diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs new file mode 100644 index 0000000000..6ffe28f12a --- /dev/null +++ b/yjit/src/stats.rs @@ -0,0 +1,1018 @@ +//! Everything related to the collection of runtime stats in YJIT +//! See the stats feature and the --yjit-stats command-line option + +#![allow(dead_code)] // Counters are only used with the stats features + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::ptr::addr_of_mut; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; +use std::collections::HashMap; + +use crate::codegen::CodegenGlobals; +use crate::core::Context; +use crate::core::for_each_iseq_payload; +use crate::cruby::*; +use crate::options::*; +use crate::yjit::yjit_enabled_p; + +/// Running total of how many ISeqs are in the system. +#[no_mangle] +pub static mut rb_yjit_live_iseq_count: u64 = 0; + +/// Monotonically increasing total of how many ISEQs were allocated +#[no_mangle] +pub static mut rb_yjit_iseq_alloc_count: u64 = 0; + +/// A middleware to count Rust-allocated bytes as yjit_alloc_size. +#[global_allocator] +static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; + +pub struct StatsAlloc { + alloc_size: AtomicUsize, +} + +unsafe impl GlobalAlloc for StatsAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); + System.alloc(layout) + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + self.alloc_size.fetch_sub(layout.size(), Ordering::SeqCst); + System.dealloc(ptr, layout) + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); + System.alloc_zeroed(layout) + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if new_size > layout.size() { + self.alloc_size.fetch_add(new_size - layout.size(), Ordering::SeqCst); + } else if new_size < layout.size() { + self.alloc_size.fetch_sub(layout.size() - new_size, Ordering::SeqCst); + } + System.realloc(ptr, layout, new_size) + } +} + +/// Mapping of C function / ISEQ name to integer indices +/// This is accessed at compilation time only (protected by a lock) +static mut CFUNC_NAME_TO_IDX: Option<HashMap<String, usize>> = None; +static mut ISEQ_NAME_TO_IDX: Option<HashMap<String, usize>> = None; + +/// Vector of call counts for each C function / ISEQ index +/// This is modified (but not resized) by JITted code +static mut CFUNC_CALL_COUNT: Option<Vec<u64>> = None; +static mut ISEQ_CALL_COUNT: Option<Vec<u64>> = None; + +/// Assign an index to a given cfunc name string +pub fn get_cfunc_idx(name: &str) -> usize { + // SAFETY: We acquire a VM lock and don't create multiple &mut references to these static mut variables. + unsafe { get_method_idx(name, &mut *addr_of_mut!(CFUNC_NAME_TO_IDX), &mut *addr_of_mut!(CFUNC_CALL_COUNT)) } +} + +/// Assign an index to a given ISEQ name string +pub fn get_iseq_idx(name: &str) -> usize { + // SAFETY: We acquire a VM lock and don't create multiple &mut references to these static mut variables. + unsafe { get_method_idx(name, &mut *addr_of_mut!(ISEQ_NAME_TO_IDX), &mut *addr_of_mut!(ISEQ_CALL_COUNT)) } +} + +fn get_method_idx( + name: &str, + method_name_to_idx: &mut Option<HashMap<String, usize>>, + method_call_count: &mut Option<Vec<u64>>, +) -> usize { + //println!("{}", name); + + let name_to_idx = method_name_to_idx.get_or_insert_with(HashMap::default); + let call_count = method_call_count.get_or_insert_with(Vec::default); + + match name_to_idx.get(name) { + Some(idx) => *idx, + None => { + let idx = name_to_idx.len(); + name_to_idx.insert(name.to_string(), idx); + + // Resize the call count vector + if idx >= call_count.len() { + call_count.resize(idx + 1, 0); + } + + idx + } + } +} + +// Increment the counter for a C function +pub extern "C" fn incr_cfunc_counter(idx: usize) { + let cfunc_call_count = unsafe { CFUNC_CALL_COUNT.as_mut().unwrap() }; + assert!(idx < cfunc_call_count.len()); + cfunc_call_count[idx] += 1; +} + +// Increment the counter for an ISEQ +pub extern "C" fn incr_iseq_counter(idx: usize) { + let iseq_call_count = unsafe { ISEQ_CALL_COUNT.as_mut().unwrap() }; + assert!(idx < iseq_call_count.len()); + iseq_call_count[idx] += 1; +} + +// YJIT exit counts for each instruction type +const VM_INSTRUCTION_SIZE_USIZE: usize = VM_INSTRUCTION_SIZE as usize; +static mut EXIT_OP_COUNT: [u64; VM_INSTRUCTION_SIZE_USIZE] = [0; VM_INSTRUCTION_SIZE_USIZE]; + +/// Global state needed for collecting backtraces of exits +pub struct YjitExitLocations { + /// Vec to hold raw_samples which represent the control frames + /// of method entries. + raw_samples: Vec<VALUE>, + /// Vec to hold line_samples which represent line numbers of + /// the iseq caller. + line_samples: Vec<i32>, + /// Number of samples skipped when sampling + skipped_samples: usize +} + +/// Private singleton instance of yjit exit locations +static mut YJIT_EXIT_LOCATIONS: Option<YjitExitLocations> = None; + +impl YjitExitLocations { + /// Initialize the yjit exit locations + pub fn init() { + // Return if --yjit-trace-exits isn't enabled + if get_option!(trace_exits).is_none() { + return; + } + + let yjit_exit_locations = YjitExitLocations { + raw_samples: Vec::new(), + line_samples: Vec::new(), + skipped_samples: 0 + }; + + // Initialize the yjit exit locations instance + unsafe { + YJIT_EXIT_LOCATIONS = Some(yjit_exit_locations); + } + } + + /// Get a mutable reference to the yjit exit locations globals instance + pub fn get_instance() -> &'static mut YjitExitLocations { + unsafe { YJIT_EXIT_LOCATIONS.as_mut().unwrap() } + } + + /// Get a mutable reference to the yjit raw samples Vec + pub fn get_raw_samples() -> &'static mut Vec<VALUE> { + &mut YjitExitLocations::get_instance().raw_samples + } + + /// Get a mutable reference to yjit the line samples Vec. + pub fn get_line_samples() -> &'static mut Vec<i32> { + &mut YjitExitLocations::get_instance().line_samples + } + + /// Get the number of samples skipped + pub fn get_skipped_samples() -> &'static mut usize { + &mut YjitExitLocations::get_instance().skipped_samples + } + + /// Mark the data stored in YjitExitLocations::get_raw_samples that needs to be used by + /// rb_yjit_add_frame. YjitExitLocations::get_raw_samples are an array of + /// VALUE pointers, exit instruction, and number of times we've seen this stack row + /// as collected by rb_yjit_record_exit_stack. + /// + /// These need to have rb_gc_mark called so they can be used by rb_yjit_add_frame. + pub fn gc_mark_raw_samples() { + // Return if YJIT is not enabled + if !yjit_enabled_p() { + return; + } + + // Return if --yjit-trace-exits isn't enabled + if get_option!(trace_exits).is_none() { + return; + } + + let mut idx: size_t = 0; + let yjit_raw_samples = YjitExitLocations::get_raw_samples(); + + while idx < yjit_raw_samples.len() as size_t { + let num = yjit_raw_samples[idx as usize]; + let mut i = 0; + idx += 1; + + // Mark the yjit_raw_samples at the given index. These represent + // the data that needs to be GC'd which are the current frames. + while i < i32::from(num) { + unsafe { rb_gc_mark(yjit_raw_samples[idx as usize]); } + i += 1; + idx += 1; + } + + // Increase index for exit instruction. + idx += 1; + // Increase index for bookkeeping value (number of times we've seen this + // row in a stack). + idx += 1; + } + } +} + +// Macro to declare the stat counters +macro_rules! make_counters { + ($($counter_name:ident,)+) => { + /// Struct containing the counter values + #[derive(Default, Debug)] + pub struct Counters { $(pub $counter_name: u64),+ } + + /// Enum to represent a counter + #[allow(non_camel_case_types)] + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub enum Counter { $($counter_name),+ } + + impl Counter { + /// Map a counter name string to a counter enum + pub fn get(name: &str) -> Option<Counter> { + match name { + $( stringify!($counter_name) => { Some(Counter::$counter_name) } ),+ + _ => None, + } + } + + /// Get a counter name string + pub fn get_name(&self) -> String { + match self { + $( Counter::$counter_name => stringify!($counter_name).to_string() ),+ + } + } + } + + /// Global counters instance, initialized to zero + pub static mut COUNTERS: Counters = Counters { $($counter_name: 0),+ }; + + /// Counter names constant + const COUNTER_NAMES: &'static [&'static str] = &[ $(stringify!($counter_name)),+ ]; + + /// Map a counter name string to a counter pointer + pub fn get_counter_ptr(name: &str) -> *mut u64 { + match name { + $( stringify!($counter_name) => { ptr_to_counter!($counter_name) } ),+ + _ => panic!() + } + } + } +} + +/// The list of counters that are available without --yjit-stats. +/// They are incremented only by `incr_counter!` and don't use `gen_counter_incr`. +pub const DEFAULT_COUNTERS: [Counter; 17] = [ + Counter::code_gc_count, + Counter::compiled_iseq_entry, + Counter::cold_iseq_entry, + Counter::compiled_iseq_count, + Counter::compiled_blockid_count, + Counter::compiled_block_count, + Counter::compiled_branch_count, + Counter::compile_time_ns, + Counter::max_inline_versions, + + Counter::invalidation_count, + Counter::invalidate_method_lookup, + Counter::invalidate_bop_redefined, + Counter::invalidate_ractor_spawn, + Counter::invalidate_constant_state_bump, + Counter::invalidate_constant_ic_fill, + Counter::invalidate_no_singleton_class, + Counter::invalidate_ep_escape, +]; + +/// Macro to increase a counter by name and count +macro_rules! incr_counter_by { + // Unsafe is ok here because options are initialized + // once before any Ruby code executes + ($counter_name:ident, $count:expr) => { + #[allow(unused_unsafe)] + { + unsafe { $crate::stats::COUNTERS.$counter_name += $count as u64 } + } + }; +} +pub(crate) use incr_counter_by; + +/// Macro to increase a counter if the given value is larger +macro_rules! incr_counter_to { + // Unsafe is ok here because options are initialized + // once before any Ruby code executes + ($counter_name:ident, $count:expr) => { + #[allow(unused_unsafe)] + { + unsafe { + $crate::stats::COUNTERS.$counter_name = u64::max( + $crate::stats::COUNTERS.$counter_name, + $count as u64, + ) + } + } + }; +} +pub(crate) use incr_counter_to; + +/// Macro to increment a counter by name +macro_rules! incr_counter { + // Unsafe is ok here because options are initialized + // once before any Ruby code executes + ($counter_name:ident) => { + #[allow(unused_unsafe)] + { + unsafe { $crate::stats::COUNTERS.$counter_name += 1 } + } + }; +} +pub(crate) use incr_counter; + +/// Macro to get a raw pointer to a given counter +macro_rules! ptr_to_counter { + ($counter_name:ident) => { + unsafe { + let ctr_ptr = std::ptr::addr_of_mut!(COUNTERS.$counter_name); + ctr_ptr + } + }; +} +pub(crate) use ptr_to_counter; + +// Declare all the counters we track +make_counters! { + yjit_insns_count, + + // Method calls that fallback to dynamic dispatch + send_singleton_class, + send_ivar_set_method, + send_zsuper_method, + send_undef_method, + send_optimized_method_block_call, + send_call_block, + send_call_kwarg, + send_call_multi_ractor, + send_cme_not_found, + send_megamorphic, + send_missing_method, + send_refined_method, + send_private_not_fcall, + send_cfunc_kw_splat_non_nil, + send_cfunc_splat_neg2, + send_cfunc_argc_mismatch, + send_cfunc_block_arg, + send_cfunc_toomany_args, + send_cfunc_tracing, + send_cfunc_splat_with_kw, + send_cfunc_splat_varg_ruby2_keywords, + send_attrset_kwargs, + send_attrset_block_arg, + send_iseq_tailcall, + send_iseq_arity_error, + send_iseq_block_arg_type, + send_iseq_clobbering_block_arg, + send_iseq_complex_discard_extras, + send_iseq_leaf_builtin_block_arg_block_param, + send_iseq_kw_splat_non_nil, + send_iseq_kwargs_mismatch, + send_iseq_has_post, + send_iseq_has_no_kw, + send_iseq_accepts_no_kwarg, + send_iseq_materialized_block, + send_iseq_splat_not_array, + send_iseq_splat_with_kw, + send_iseq_missing_optional_kw, + send_iseq_too_many_kwargs, + send_not_implemented_method, + send_getter_arity, + send_getter_block_arg, + send_args_splat_attrset, + send_args_splat_bmethod, + send_args_splat_aref, + send_args_splat_aset, + send_args_splat_opt_call, + send_iseq_splat_arity_error, + send_splat_too_long, + send_send_wrong_args, + send_send_null_mid, + send_send_null_cme, + send_send_nested, + send_send_attr_reader, + send_send_attr_writer, + send_iseq_has_rest_and_captured, + send_iseq_has_kwrest_and_captured, + send_iseq_has_rest_and_kw_supplied, + send_iseq_has_rest_opt_and_block, + send_bmethod_ractor, + send_bmethod_block_arg, + send_optimized_block_arg, + + invokesuper_defined_class_mismatch, + invokesuper_kw_splat, + invokesuper_kwarg, + invokesuper_megamorphic, + invokesuper_no_cme, + invokesuper_no_me, + invokesuper_not_iseq_or_cfunc, + invokesuper_refinement, + invokesuper_singleton_class, + + invokeblock_megamorphic, + invokeblock_none, + invokeblock_iseq_arg0_optional, + invokeblock_iseq_arg0_args_splat, + invokeblock_iseq_arg0_not_array, + invokeblock_iseq_arg0_wrong_len, + invokeblock_iseq_not_inlined, + invokeblock_ifunc_args_splat, + invokeblock_ifunc_kw_splat, + invokeblock_proc, + invokeblock_symbol, + + // Method calls that exit to the interpreter + guard_send_block_arg_type, + guard_send_getter_splat_non_empty, + guard_send_klass_megamorphic, + guard_send_se_cf_overflow, + guard_send_se_protected_check_failed, + guard_send_splatarray_length_not_equal, + guard_send_splatarray_last_ruby2_keywords, + guard_send_splat_not_array, + guard_send_send_name_chain, + guard_send_iseq_has_rest_and_splat_too_few, + guard_send_is_a_class_mismatch, + guard_send_instance_of_class_mismatch, + guard_send_interrupted, + guard_send_not_fixnums, + guard_send_not_fixnum_or_flonum, + guard_send_not_string, + guard_send_respond_to_mid_mismatch, + + guard_send_cfunc_bad_splat_vargs, + + guard_invokesuper_me_changed, + + guard_invokeblock_tag_changed, + guard_invokeblock_iseq_block_changed, + + traced_cfunc_return, + + leave_se_interrupt, + leave_interp_return, + + getivar_megamorphic, + getivar_not_heap, + + setivar_not_heap, + setivar_frozen, + setivar_megamorphic, + + definedivar_not_heap, + definedivar_megamorphic, + + setlocal_wb_required, + + invokebuiltin_too_many_args, + + opt_plus_overflow, + opt_minus_overflow, + opt_mult_overflow, + + opt_succ_not_fixnum, + opt_succ_overflow, + + opt_mod_zero, + opt_div_zero, + + lshift_amount_changed, + lshift_overflow, + + rshift_amount_changed, + + opt_aref_argc_not_one, + opt_aref_arg_not_fixnum, + opt_aref_not_array, + opt_aref_not_hash, + + opt_aset_not_array, + opt_aset_not_fixnum, + opt_aset_not_hash, + + opt_aref_with_qundef, + + opt_case_dispatch_megamorphic, + + opt_getconstant_path_ic_miss, + opt_getconstant_path_multi_ractor, + + expandarray_splat, + expandarray_postarg, + expandarray_not_array, + expandarray_to_ary, + expandarray_chain_max_depth, + + // getblockparam + gbp_wb_required, + + // getblockparamproxy + gbpp_unsupported_type, + gbpp_block_param_modified, + gbpp_block_handler_not_none, + gbpp_block_handler_not_iseq, + gbpp_block_handler_not_proc, + + branchif_interrupted, + branchunless_interrupted, + branchnil_interrupted, + jump_interrupted, + + objtostring_not_string, + + getbyte_idx_not_fixnum, + getbyte_idx_negative, + getbyte_idx_out_of_bounds, + + splatkw_not_hash, + splatkw_not_nil, + + binding_allocations, + binding_set, + + compiled_iseq_entry, + cold_iseq_entry, + compiled_iseq_count, + compiled_blockid_count, + compiled_block_count, + compiled_branch_count, + compile_time_ns, + compilation_failure, + block_next_count, + defer_count, + defer_empty_count, + branch_insn_count, + branch_known_count, + max_inline_versions, + + freed_iseq_count, + + exit_from_branch_stub, + + invalidation_count, + invalidate_method_lookup, + invalidate_bop_redefined, + invalidate_ractor_spawn, + invalidate_constant_state_bump, + invalidate_constant_ic_fill, + invalidate_no_singleton_class, + invalidate_ep_escape, + + // Currently, it's out of the ordinary (might be impossible) for YJIT to leave gaps in + // executable memory, so this should be 0. + exec_mem_non_bump_alloc, + + code_gc_count, + + num_gc_obj_refs, + + num_send, + num_send_known_class, + num_send_polymorphic, + num_send_x86_rel32, + num_send_x86_reg, + num_send_dynamic, + num_send_cfunc, + num_send_cfunc_inline, + num_send_iseq, + num_send_iseq_leaf, + num_send_iseq_inline, + + num_getivar_megamorphic, + num_setivar_megamorphic, + num_opt_case_dispatch_megamorphic, + + num_throw, + num_throw_break, + num_throw_retry, + num_throw_return, + + num_lazy_frame_check, + num_lazy_frame_push, + lazy_frame_count, + lazy_frame_failure, + + iseq_stack_too_large, + iseq_too_long, + + temp_reg_opnd, + temp_mem_opnd, + temp_spill, +} + +//=========================================================================== + +/// Primitive called in yjit.rb +/// Check if stats generation is enabled +#[no_mangle] +pub extern "C" fn rb_yjit_stats_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + + if get_option!(gen_stats) { + return Qtrue; + } else { + return Qfalse; + } +} + +/// Primitive called in yjit.rb +/// Check if stats generation should print at exit +#[no_mangle] +pub extern "C" fn rb_yjit_print_stats_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + if yjit_enabled_p() && get_option!(print_stats) { + return Qtrue; + } else { + return Qfalse; + } +} + +/// Primitive called in yjit.rb. +/// Export all YJIT statistics as a Ruby hash. +#[no_mangle] +pub extern "C" fn rb_yjit_get_stats(_ec: EcPtr, _ruby_self: VALUE, context: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || rb_yjit_gen_stats_dict(context == Qtrue)) +} + +/// Primitive called in yjit.rb +/// +/// Check if trace_exits generation is enabled. Requires the stats feature +/// to be enabled. +#[no_mangle] +pub extern "C" fn rb_yjit_trace_exit_locations_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + if get_option!(trace_exits).is_some() { + return Qtrue; + } + + return Qfalse; +} + +/// Call the C function to parse the raw_samples and line_samples +/// into raw, lines, and frames hash for RubyVM::YJIT.exit_locations. +#[no_mangle] +pub extern "C" fn rb_yjit_get_exit_locations(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + // Return if YJIT is not enabled + if !yjit_enabled_p() { + return Qnil; + } + + // Return if --yjit-trace-exits isn't enabled + if get_option!(trace_exits).is_none() { + return Qnil; + } + + // If the stats feature is enabled, pass yjit_raw_samples and yjit_line_samples + // to the C function called rb_yjit_exit_locations_dict for parsing. + let yjit_raw_samples = YjitExitLocations::get_raw_samples(); + let yjit_line_samples = YjitExitLocations::get_line_samples(); + + // Assert that the two Vec's are the same length. If they aren't + // equal something went wrong. + assert_eq!(yjit_raw_samples.len(), yjit_line_samples.len()); + + // yjit_raw_samples and yjit_line_samples are the same length so + // pass only one of the lengths in the C function. + let samples_len = yjit_raw_samples.len() as i32; + + unsafe { + rb_yjit_exit_locations_dict(yjit_raw_samples.as_mut_ptr(), yjit_line_samples.as_mut_ptr(), samples_len) + } +} + +/// Increment a counter by name from the CRuby side +/// Warning: this is not fast because it requires a hash lookup, so don't use in tight loops +#[no_mangle] +pub extern "C" fn rb_yjit_incr_counter(counter_name: *const std::os::raw::c_char) { + use std::ffi::CStr; + let counter_name = unsafe { CStr::from_ptr(counter_name).to_str().unwrap() }; + let counter_ptr = get_counter_ptr(counter_name); + unsafe { *counter_ptr += 1 }; +} + +/// Export all YJIT statistics as a Ruby hash. +fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { + // If YJIT is not enabled, return Qnil + if !yjit_enabled_p() { + return Qnil; + } + + macro_rules! hash_aset_usize { + ($hash:ident, $counter_name:expr, $value:expr) => { + let key = rust_str_to_sym($counter_name); + let value = VALUE::fixnum_from_usize($value); + rb_hash_aset($hash, key, value); + } + } + + let hash = unsafe { rb_hash_new() }; + + unsafe { + // Get the inline and outlined code blocks + let cb = CodegenGlobals::get_inline_cb(); + let ocb = CodegenGlobals::get_outlined_cb(); + + // Inline code size + hash_aset_usize!(hash, "inline_code_size", cb.code_size()); + + // Outlined code size + hash_aset_usize!(hash, "outlined_code_size", ocb.unwrap().code_size()); + + // GCed pages + let freed_page_count = cb.num_freed_pages(); + hash_aset_usize!(hash, "freed_page_count", freed_page_count); + + // GCed code size + hash_aset_usize!(hash, "freed_code_size", freed_page_count * cb.page_size()); + + // Live pages + hash_aset_usize!(hash, "live_page_count", cb.num_mapped_pages() - freed_page_count); + + // Size of memory region allocated for JIT code + hash_aset_usize!(hash, "code_region_size", cb.mapped_region_size()); + + // Rust global allocations in bytes + hash_aset_usize!(hash, "yjit_alloc_size", GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst)); + + // `context` is true at RubyVM::YJIT._print_stats for --yjit-stats. It's false by default + // for RubyVM::YJIT.runtime_stats because counting all Contexts could be expensive. + if context { + let live_context_count = get_live_context_count(); + let context_size = std::mem::size_of::<Context>(); + hash_aset_usize!(hash, "live_context_count", live_context_count); + hash_aset_usize!(hash, "live_context_size", live_context_count * context_size); + } + + // VM instructions count + hash_aset_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize); + + hash_aset_usize!(hash, "live_iseq_count", rb_yjit_live_iseq_count as usize); + hash_aset_usize!(hash, "iseq_alloc_count", rb_yjit_iseq_alloc_count as usize); + } + + // If we're not generating stats, put only default counters + if !get_option!(gen_stats) { + for counter in DEFAULT_COUNTERS { + // Get the counter value + let counter_ptr = get_counter_ptr(&counter.get_name()); + let counter_val = unsafe { *counter_ptr }; + + // Put counter into hash + let key = rust_str_to_sym(&counter.get_name()); + let value = VALUE::fixnum_from_usize(counter_val as usize); + unsafe { rb_hash_aset(hash, key, value); } + } + + return hash; + } + + unsafe { + // Indicate that the complete set of stats is available + rb_hash_aset(hash, rust_str_to_sym("all_stats"), Qtrue); + + // For each counter we track + for counter_name in COUNTER_NAMES { + // Get the counter value + let counter_ptr = get_counter_ptr(counter_name); + let counter_val = *counter_ptr; + + // Put counter into hash + let key = rust_str_to_sym(counter_name); + let value = VALUE::fixnum_from_usize(counter_val as usize); + rb_hash_aset(hash, key, value); + } + + // For each entry in exit_op_count, add a stats entry with key "exit_INSTRUCTION_NAME" + // and the value is the count of side exits for that instruction. + for op_idx in 0..VM_INSTRUCTION_SIZE_USIZE { + let op_name = insn_name(op_idx); + let key_string = "exit_".to_owned() + &op_name; + let key = rust_str_to_sym(&key_string); + let value = VALUE::fixnum_from_usize(EXIT_OP_COUNT[op_idx] as usize); + rb_hash_aset(hash, key, value); + } + + fn set_call_counts( + calls_hash: VALUE, + method_name_to_idx: &mut Option<HashMap<String, usize>>, + method_call_count: &mut Option<Vec<u64>>, + ) { + if let (Some(name_to_idx), Some(call_counts)) = (method_name_to_idx, method_call_count) { + for (name, idx) in name_to_idx { + let count = call_counts[*idx]; + let key = rust_str_to_sym(name); + let value = VALUE::fixnum_from_usize(count as usize); + unsafe { rb_hash_aset(calls_hash, key, value); } + } + } + } + + // Create a hash for the cfunc call counts + let cfunc_calls = rb_hash_new(); + rb_hash_aset(hash, rust_str_to_sym("cfunc_calls"), cfunc_calls); + set_call_counts(cfunc_calls, &mut *addr_of_mut!(CFUNC_NAME_TO_IDX), &mut *addr_of_mut!(CFUNC_CALL_COUNT)); + + // Create a hash for the ISEQ call counts + let iseq_calls = rb_hash_new(); + rb_hash_aset(hash, rust_str_to_sym("iseq_calls"), iseq_calls); + set_call_counts(iseq_calls, &mut *addr_of_mut!(ISEQ_NAME_TO_IDX), &mut *addr_of_mut!(ISEQ_CALL_COUNT)); + } + + hash +} + +fn get_live_context_count() -> usize { + let mut count = 0; + for_each_iseq_payload(|iseq_payload| { + for blocks in iseq_payload.version_map.iter() { + for block in blocks.iter() { + count += unsafe { block.as_ref() }.get_ctx_count(); + } + } + for block in iseq_payload.dead_blocks.iter() { + count += unsafe { block.as_ref() }.get_ctx_count(); + } + }); + count +} + +/// Record the backtrace when a YJIT exit occurs. This functionality requires +/// that the stats feature is enabled as well as the --yjit-trace-exits option. +/// +/// This function will fill two Vec's in YjitExitLocations to record the raw samples +/// and line samples. Their length should be the same, however the data stored in +/// them is different. +#[no_mangle] +pub extern "C" fn rb_yjit_record_exit_stack(_exit_pc: *const VALUE) +{ + // Return if YJIT is not enabled + if !yjit_enabled_p() { + return; + } + + // Return if --yjit-trace-exits isn't enabled + if get_option!(trace_exits).is_none() { + return; + } + + if get_option!(trace_exits_sample_rate) > 0 { + if get_option!(trace_exits_sample_rate) <= *YjitExitLocations::get_skipped_samples() { + YjitExitLocations::get_instance().skipped_samples = 0; + } else { + YjitExitLocations::get_instance().skipped_samples += 1; + return; + } + } + + // rb_vm_insn_addr2opcode won't work in cargo test --all-features + // because it's a C function. Without insn call, this function is useless + // so wrap the whole thing in a not test check. + #[cfg(not(test))] + { + // Get the opcode from the encoded insn handler at this PC + let insn = unsafe { rb_vm_insn_addr2opcode((*_exit_pc).as_ptr()) }; + + // Use the same buffer size as Stackprof. + const BUFF_LEN: usize = 2048; + + // Create 2 array buffers to be used to collect frames and lines. + let mut frames_buffer = [VALUE(0_usize); BUFF_LEN]; + let mut lines_buffer = [0; BUFF_LEN]; + + // Records call frame and line information for each method entry into two + // temporary buffers. Returns the number of times we added to the buffer (ie + // the length of the stack). + // + // Call frame info is stored in the frames_buffer, line number information + // in the lines_buffer. The first argument is the start point and the second + // argument is the buffer limit, set at 2048. + let stack_length = unsafe { rb_profile_frames(0, BUFF_LEN as i32, frames_buffer.as_mut_ptr(), lines_buffer.as_mut_ptr()) }; + let samples_length = (stack_length as usize) + 3; + + let yjit_raw_samples = YjitExitLocations::get_raw_samples(); + let yjit_line_samples = YjitExitLocations::get_line_samples(); + + // If yjit_raw_samples is less than or equal to the current length of the samples + // we might have seen this stack trace previously. + if yjit_raw_samples.len() >= samples_length { + let prev_stack_len_index = yjit_raw_samples.len() - samples_length; + let prev_stack_len = i64::from(yjit_raw_samples[prev_stack_len_index]); + let mut idx = stack_length - 1; + let mut prev_frame_idx = 0; + let mut seen_already = true; + + // If the previous stack length and current stack length are equal, + // loop and compare the current frame to the previous frame. If they are + // not equal, set seen_already to false and break out of the loop. + if prev_stack_len == stack_length as i64 { + while idx >= 0 { + let current_frame = frames_buffer[idx as usize]; + let prev_frame = yjit_raw_samples[prev_stack_len_index + prev_frame_idx + 1]; + + // If the current frame and previous frame are not equal, set + // seen_already to false and break out of the loop. + if current_frame != prev_frame { + seen_already = false; + break; + } + + idx -= 1; + prev_frame_idx += 1; + } + + // If we know we've seen this stack before, increment the counter by 1. + if seen_already { + let prev_idx = yjit_raw_samples.len() - 1; + let prev_count = i64::from(yjit_raw_samples[prev_idx]); + let new_count = prev_count + 1; + + yjit_raw_samples[prev_idx] = VALUE(new_count as usize); + yjit_line_samples[prev_idx] = new_count as i32; + + return; + } + } + } + + yjit_raw_samples.push(VALUE(stack_length as usize)); + yjit_line_samples.push(stack_length); + + let mut idx = stack_length - 1; + + while idx >= 0 { + let frame = frames_buffer[idx as usize]; + let line = lines_buffer[idx as usize]; + + yjit_raw_samples.push(frame); + yjit_line_samples.push(line); + + idx -= 1; + } + + // Push the insn value into the yjit_raw_samples Vec. + yjit_raw_samples.push(VALUE(insn as usize)); + + // We don't know the line + yjit_line_samples.push(0); + + // Push number of times seen onto the stack, which is 1 + // because it's the first time we've seen it. + yjit_raw_samples.push(VALUE(1_usize)); + yjit_line_samples.push(1); + } +} + +/// Primitive called in yjit.rb. Zero out all the counters. +#[no_mangle] +pub extern "C" fn rb_yjit_reset_stats_bang(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + unsafe { + EXIT_OP_COUNT = [0; VM_INSTRUCTION_SIZE_USIZE]; + COUNTERS = Counters::default(); + } + + return Qnil; +} + +#[no_mangle] +pub extern "C" fn rb_yjit_collect_binding_alloc() { + incr_counter!(binding_allocations); +} + +#[no_mangle] +pub extern "C" fn rb_yjit_collect_binding_set() { + incr_counter!(binding_set); +} + +#[no_mangle] +pub extern "C" fn rb_yjit_count_side_exit_op(exit_pc: *const VALUE) -> *const VALUE { + #[cfg(not(test))] + unsafe { + // Get the opcode from the encoded insn handler at this PC + let opcode = rb_vm_insn_addr2opcode((*exit_pc).as_ptr()); + + // Increment the exit op count for this opcode + EXIT_OP_COUNT[opcode as usize] += 1; + }; + + // This function must return exit_pc! + return exit_pc; +} + +/// Measure the time taken by func() and add that to yjit_compile_time. +pub fn with_compile_time<F, R>(func: F) -> R where F: FnOnce() -> R { + let start = Instant::now(); + let ret = func(); + let nanos = Instant::now().duration_since(start).as_nanos(); + incr_counter_by!(compile_time_ns, nanos); + ret +} |