diff options
Diffstat (limited to 'gc/mmtk')
| -rw-r--r-- | gc/mmtk/.gitignore | 1 | ||||
| -rw-r--r-- | gc/mmtk/Cargo.lock | 1108 | ||||
| -rw-r--r-- | gc/mmtk/Cargo.toml | 42 | ||||
| -rw-r--r-- | gc/mmtk/cbindgen.toml | 36 | ||||
| -rw-r--r-- | gc/mmtk/depend | 18 | ||||
| -rw-r--r-- | gc/mmtk/extconf.rb | 24 | ||||
| -rw-r--r-- | gc/mmtk/mmtk.c | 1658 | ||||
| -rw-r--r-- | gc/mmtk/mmtk.h | 175 | ||||
| -rw-r--r-- | gc/mmtk/src/abi.rs | 335 | ||||
| -rw-r--r-- | gc/mmtk/src/active_plan.rs | 56 | ||||
| -rw-r--r-- | gc/mmtk/src/api.rs | 551 | ||||
| -rw-r--r-- | gc/mmtk/src/binding.rs | 129 | ||||
| -rw-r--r-- | gc/mmtk/src/collection.rs | 122 | ||||
| -rw-r--r-- | gc/mmtk/src/heap/cpu_heap_trigger.rs | 370 | ||||
| -rw-r--r-- | gc/mmtk/src/heap/mod.rs | 9 | ||||
| -rw-r--r-- | gc/mmtk/src/heap/ruby_heap_trigger.rs | 105 | ||||
| -rw-r--r-- | gc/mmtk/src/lib.rs | 161 | ||||
| -rw-r--r-- | gc/mmtk/src/object_model.rs | 124 | ||||
| -rw-r--r-- | gc/mmtk/src/pinning_registry.rs | 187 | ||||
| -rw-r--r-- | gc/mmtk/src/reference_glue.rs | 26 | ||||
| -rw-r--r-- | gc/mmtk/src/scanning.rs | 291 | ||||
| -rw-r--r-- | gc/mmtk/src/utils.rs | 161 | ||||
| -rw-r--r-- | gc/mmtk/src/weak_proc.rs | 328 |
23 files changed, 6017 insertions, 0 deletions
diff --git a/gc/mmtk/.gitignore b/gc/mmtk/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/gc/mmtk/.gitignore @@ -0,0 +1 @@ +target diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock new file mode 100644 index 0000000000..910048fa80 --- /dev/null +++ b/gc/mmtk/Cargo.lock @@ -0,0 +1,1108 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-traits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f750b93bd1b739cf9ddf85f8fe7c97a4a62c60ccf8b6f232514bd9103bedc" +dependencies = [ + "cfg-if", + "rustc_version", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" +dependencies = [ + "git2", +] + +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "cc" +version = "1.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "delegate" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "downcast-rs" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8" +dependencies = [ + "idna_mapping", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna_mapping" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792" +dependencies = [ + "unicode-joining-type", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.1", + "libc", + "windows-sys", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "jiff" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mmtk" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "atomic", + "atomic-traits", + "atomic_refcell", + "built", + "bytemuck", + "bytemuck_derive", + "cfg-if", + "crossbeam", + "delegate", + "downcast-rs", + "enum-map", + "env_logger", + "idna_adapter", + "is-terminal", + "itertools", + "lazy_static", + "libc", + "log", + "memoffset", + "mmtk-macros", + "num-traits", + "num_cpus", + "portable-atomic", + "probe", + "rayon-core", + "regex", + "rustversion", + "spin", + "static_assertions", + "strum", + "strum_macros", + "sysinfo 0.33.1", +] + +[[package]] +name = "mmtk-macros" +version = "0.31.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "mmtk_ruby" +version = "0.1.0" +dependencies = [ + "atomic_refcell", + "env_logger", + "libc", + "log", + "mmtk", + "once_cell", + "probe", + "sysinfo 0.32.1", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "probe" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e2d2444b730c8f027344c60f9e1f1554d7a3342df9bdd425142ed119a6e5a3" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" + +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.101", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-joining-type" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml new file mode 100644 index 0000000000..d856122900 --- /dev/null +++ b/gc/mmtk/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "mmtk_ruby" +version = "0.1.0" +authors = [] +edition = "2021" + +[lib] +name = "mmtk_ruby" +crate-type = ["cdylib", "staticlib"] + +[profile.release] +lto = true + +[dependencies] +libc = "0.2" +log = "0.4.14" +env_logger = "0.11.3" +once_cell = "1.17.0" +atomic_refcell = "0.1.9" +probe = "0.5" +sysinfo = "0.32.0" + +[dependencies.mmtk] +features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"] + +# Uncomment the following lines to use mmtk-core from the official repository. +git = "https://github.com/mmtk/mmtk-core.git" +rev = "c6317a3f1c262e33fc2e427e4cc999c17bcc4791" + +# Uncomment the following line to use mmtk-core from a local repository. +# path = "../../../mmtk-core" + +[features] +default = [] + +# When moving an object, clear its original copy. +clear_old_copy = [] + +# Enable extra assertions in release build. For debugging. +extra_assert = [] + +[workspace] diff --git a/gc/mmtk/cbindgen.toml b/gc/mmtk/cbindgen.toml new file mode 100644 index 0000000000..b99c30efc8 --- /dev/null +++ b/gc/mmtk/cbindgen.toml @@ -0,0 +1,36 @@ +language = "C" + +include_guard = "MMTK_H" + +autogen_warning = "/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */" + +tab_width = 4 + +usize_is_size_t = true + +after_includes = """ + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; +""" + +[export] +exclude = ["RubyMutator"] +prefix = "MMTk_" + +[export.rename] +"MMTKBuilder" = "Builder" +"RubyMutator" = "Mutator" diff --git a/gc/mmtk/depend b/gc/mmtk/depend new file mode 100644 index 0000000000..77b229af36 --- /dev/null +++ b/gc/mmtk/depend @@ -0,0 +1,18 @@ +$(TARGET_SO): $(MMTK_BUILD)/$(LIBMMTK_RUBY) + +# Add the `libmmtk_ruby.a` target to run `cargo build` + +release/$(LIBMMTK_RUBY) debug/$(LIBMMTK_RUBY): $(RUSTSRCS) $(srcdir)/Cargo.toml $(srcdir)/Cargo.toml + +release/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml --release + +debug/$(LIBMMTK_RUBY): + CARGO_TARGET_DIR="." cargo build --manifest-path=$(srcdir)/Cargo.toml + +clean: clean-mmtk + +.PHONY: clean-mmtk +clean-mmtk: + -$(Q)$(RM_RF) debug release + -$(Q)$(RM) .rustc_info.json diff --git a/gc/mmtk/extconf.rb b/gc/mmtk/extconf.rb new file mode 100644 index 0000000000..c0e788037e --- /dev/null +++ b/gc/mmtk/extconf.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require_relative "../extconf_base" + +# Statically link `libmmtk_ruby.a` +$LIBS << " $(MMTK_BUILD)/$(LIBMMTK_RUBY)" + +rustsrcs = Dir.glob("src/*.rs", base: __dir__).map {|s| "$(srcdir)/#{s}"} + +create_gc_makefile("mmtk") do |makefile| + [ + *makefile, + + <<~MAKEFILE, + MMTK_BUILD = debug + LIBMMTK_RUBY = libmmtk_ruby.#$LIBEXT + RUSTSRCS = #{rustsrcs.join(" \\\n\t ")} + + ifeq ($(MMTK_BUILD), debug) + CPPFLAGS += -DMMTK_DEBUG + endif + MAKEFILE + ] +end diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c new file mode 100644 index 0000000000..8be69b4fe6 --- /dev/null +++ b/gc/mmtk/mmtk.c @@ -0,0 +1,1658 @@ +#include <pthread.h> +#include <stdbool.h> + +#include "ruby/assert.h" +#include "ruby/atomic.h" +#include "ruby/debug.h" + +#include "gc/gc.h" +#include "gc/gc_impl.h" +#include "gc/mmtk/mmtk.h" + +#include "ccan/list/list.h" +#include "darray.h" + +#ifdef __APPLE__ +#include <sys/sysctl.h> +#endif + +struct objspace { + bool measure_gc_time; + bool gc_stress; + + size_t gc_count; + size_t moving_gc_count; + size_t total_gc_time; + size_t total_allocated_objects; + + st_table *finalizer_table; + struct MMTk_final_job *finalizer_jobs; + rb_postponed_job_handle_t finalizer_postponed_job; + + struct ccan_list_head ractor_caches; + unsigned long live_ractor_cache_count; + + pthread_mutex_t mutex; + rb_atomic_t mutator_blocking_count; + bool world_stopped; + pthread_cond_t cond_world_stopped; + pthread_cond_t cond_world_started; + size_t start_the_world_count; + + pthread_mutex_t event_hook_mutex; + + struct { + bool gc_thread_crashed; + char crash_msg[256]; + } crash_context; + + struct rb_gc_vm_context vm_context; + + unsigned int fork_hook_vm_lock_lev; +}; + +#define OBJ_FREE_BUF_CAPACITY 128 + +struct MMTk_ractor_cache { + struct ccan_list_node list_node; + + MMTk_Mutator *mutator; + bool gc_mutator_p; + + MMTk_BumpPointer *bump_pointer; + + MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_parallel_count; + MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY]; + size_t obj_free_non_parallel_count; +}; + +struct MMTk_final_job { + struct MMTk_final_job *next; + enum { + MMTK_FINAL_JOB_DFREE, + MMTK_FINAL_JOB_FINALIZE, + } kind; + union { + struct { + void (*func)(void *); + void *data; + } dfree; + struct { + /* HACK: we store the object ID on the 0th element of this array. */ + VALUE finalizer_array; + } finalize; + } as; +}; + +#ifdef RB_THREAD_LOCAL_SPECIFIER +RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls; + +RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object; +#else +# error We currently need language-supported TLS +#endif + +#ifdef MMTK_DEBUG +# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__)) +#else +# define MMTK_ASSERT(expr, ...) ((void)0) +#endif + +#include <pthread.h> + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static void +rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls) +{ + rb_mmtk_gc_thread_tls = gc_thread_tls; +} + +static bool +rb_mmtk_is_mutator(void) +{ + return ruby_native_thread_p(); +} + +static void +rb_mmtk_stop_the_world(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + while (!objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_stopped, &objspace->mutex); + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void +rb_mmtk_resume_mutators(bool current_gc_may_move) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + objspace->world_stopped = false; + objspace->gc_count++; + if (current_gc_may_move) objspace->moving_gc_count++; + pthread_cond_broadcast(&objspace->cond_world_started); + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } +} + +static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache); + +static void +rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + size_t starting_gc_count = objspace->gc_count; + RUBY_ATOMIC_INC(objspace->mutator_blocking_count); + int lock_lev = RB_GC_VM_LOCK(); + RUBY_ATOMIC_DEC(objspace->mutator_blocking_count); + int err; + if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err)); + } + + if (objspace->gc_count == starting_gc_count) { + rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START); + + rb_gc_initialize_vm_context(&objspace->vm_context); + + mutator->gc_mutator_p = true; + + struct timespec gc_start_time; + if (objspace->measure_gc_time) { + clock_gettime(CLOCK_MONOTONIC, &gc_start_time); + } + + rb_gc_save_machine_context(); + + rb_gc_vm_barrier(); + + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + objspace->world_stopped = true; + + pthread_cond_broadcast(&objspace->cond_world_stopped); + + // Wait for GC end + while (objspace->world_stopped) { + pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex); + } + + if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) { + rb_bug("%s", objspace->crash_context.crash_msg); + } + + if (objspace->measure_gc_time) { + struct timespec gc_end_time; + clock_gettime(CLOCK_MONOTONIC, &gc_end_time); + + objspace->total_gc_time += + (gc_end_time.tv_sec - gc_start_time.tv_sec) * (1000 * 1000 * 1000) + + (gc_end_time.tv_nsec - gc_start_time.tv_nsec); + } + } + + if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) { + rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err)); + } + RB_GC_VM_UNLOCK(lock_lev); +} + +static void +rb_mmtk_before_updating_jit_code(void) +{ + rb_gc_before_updating_jit_code(); +} + +static void +rb_mmtk_after_updating_jit_code(void) +{ + rb_gc_after_updating_jit_code(); +} + +static size_t +rb_mmtk_number_of_mutators(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + return objspace->live_ractor_cache_count; +} + +static void +rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data) +{ + struct objspace *objspace = rb_gc_get_objspace(); + struct MMTk_ractor_cache *ractor_cache; + + ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) { + visit_mutator(ractor_cache->mutator, data); + } +} + +static void +rb_mmtk_scan_gc_roots(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + rb_gc_mark_roots(objspace, NULL); +} + +static int +pin_value(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_and_pin((void *)data, (VALUE)value); + + return ST_CONTINUE; +} + +static void +rb_mmtk_scan_objspace(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + if (objspace->finalizer_table != NULL) { + st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace); + } + + struct MMTk_final_job *job = objspace->finalizer_jobs; + while (job != NULL) { + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + break; + case MMTK_FINAL_JOB_FINALIZE: + rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array); + break; + default: + rb_bug("rb_mmtk_scan_objspace: unknown final job type %d", job->kind); + } + + job = job->next; + } +} + +static void +rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to) +{ + rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to); +} + +static void +rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object) +{ + VALUE object = (VALUE)mmtk_object; + + if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) { + marking_parent_object = object; + rb_gc_update_object_references(rb_gc_get_objspace(), object); + marking_parent_object = 0; + } +} + +static void +rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object) +{ + marking_parent_object = (VALUE)object; + rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object); + marking_parent_object = 0; +} + +static void +rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving) +{ + VALUE object = (VALUE)mmtk_object; + + marking_parent_object = object; + + rb_gc_handle_weak_references(object); + + if (moving) { + rb_gc_update_object_references(rb_gc_get_objspace(), object); + } + + marking_parent_object = 0; +} + +static void +rb_mmtk_call_obj_free(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + struct objspace *objspace = rb_gc_get_objspace(); + + if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) { + pthread_mutex_lock(&objspace->event_hook_mutex); + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ); + pthread_mutex_unlock(&objspace->event_hook_mutex); + } + + if (RB_UNLIKELY(rb_gc_obj_needs_cleanup_p(obj))) { + rb_gc_obj_free(objspace, obj); + } + +#ifdef MMTK_DEBUG + memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj)); +#endif +} + +static size_t +rb_mmtk_vm_live_bytes(void) +{ + return 0; +} + +static void +make_final_job(struct objspace *objspace, VALUE obj, VALUE table) +{ + MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY); + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->next = objspace->finalizer_jobs; + job->kind = MMTK_FINAL_JOB_FINALIZE; + job->as.finalize.finalizer_array = table; + + objspace->finalizer_jobs = job; +} + +static int +rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error) +{ + MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value)); + MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY); + + struct objspace *objspace = (struct objspace *)data; + + if (mmtk_is_reachable((MMTk_ObjectReference)key)) { + VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false); + + MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE)); + + if (new_key_location != key) { + return ST_REPLACE; + } + } + else { + make_final_job(objspace, (VALUE)key, (VALUE)value); + + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + + return ST_DELETE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing) +{ + *key = rb_mmtk_call_object_closure((VALUE)*key, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_finalizer_table(void) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + st_foreach_with_replace( + objspace->finalizer_table, + rb_mmtk_update_finalizer_table_i, + rb_mmtk_update_finalizer_table_replace_i, + (st_data_t)objspace + ); +} + +static int +rb_mmtk_global_tables_count(void) +{ + return RB_GC_VM_WEAK_TABLE_COUNT; +} + +static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin); + +static int +rb_mmtk_update_global_tables_i(VALUE val, void *data) +{ + if (!mmtk_is_reachable((MMTk_ObjectReference)val)) { + return ST_DELETE; + } + + // TODO: check only if in moving GC + if (rb_mmtk_call_object_closure(val, false) != val) { + return ST_REPLACE; + } + + return ST_CONTINUE; +} + +static int +rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data) +{ + // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice + *ptr = rb_mmtk_call_object_closure(*ptr, false); + + return ST_CONTINUE; +} + +static void +rb_mmtk_update_global_tables(int table, bool moving) +{ + MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT); + + rb_gc_vm_weak_table_foreach( + rb_mmtk_update_global_tables_i, + rb_mmtk_update_global_tables_replace_i, + NULL, + !moving, + (enum rb_gc_vm_weak_tables)table + ); +} + +static bool +rb_mmtk_special_const_p(MMTk_ObjectReference object) +{ + VALUE obj = (VALUE)object; + + return RB_SPECIAL_CONST_P(obj); +} + +RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2) +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_bug(const char *msg, ...) +{ + struct objspace *objspace = rb_gc_get_objspace(); + + objspace->crash_context.gc_thread_crashed = true; + + va_list args; + va_start(args, msg); + vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args); + va_end(args); + + fprintf(stderr, "-- GC thread backtrace " + "-------------------------------------------\n"); + rb_gc_print_backtrace(); + fprintf(stderr, "\n"); + + rb_mmtk_resume_mutators(false); + + sleep(5); + + rb_bug("rb_mmtk_gc_thread_bug"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_gc_thread_panic_handler(void) +{ + rb_mmtk_gc_thread_bug("MMTk GC thread panicked"); +} + +RBIMPL_ATTR_NORETURN() +static void +rb_mmtk_mutator_thread_panic_handler(void) +{ + rb_bug("Ruby mutator thread panicked"); +} + +// Bootup +MMTk_RubyUpcalls ruby_upcalls = { + rb_mmtk_init_gc_worker_thread, + rb_mmtk_is_mutator, + rb_mmtk_stop_the_world, + rb_mmtk_resume_mutators, + rb_mmtk_block_for_gc, + rb_mmtk_before_updating_jit_code, + rb_mmtk_after_updating_jit_code, + rb_mmtk_number_of_mutators, + rb_mmtk_get_mutators, + rb_mmtk_scan_gc_roots, + rb_mmtk_scan_objspace, + rb_mmtk_move_obj_during_marking, + rb_mmtk_update_object_references, + rb_mmtk_call_gc_mark_children, + rb_mmtk_handle_weak_references, + rb_mmtk_call_obj_free, + rb_mmtk_vm_live_bytes, + rb_mmtk_update_global_tables, + rb_mmtk_global_tables_count, + rb_mmtk_update_finalizer_table, + rb_mmtk_special_const_p, + rb_mmtk_mutator_thread_panic_handler, + rb_mmtk_gc_thread_panic_handler, +}; + +// Use max 80% of the available memory by default for MMTk +#define RB_MMTK_HEAP_LIMIT_PERC 80 +#define RB_MMTK_DEFAULT_HEAP_MIN (1024 * 1024) +#define RB_MMTK_DEFAULT_HEAP_MAX (rb_mmtk_system_physical_memory() / 100 * RB_MMTK_HEAP_LIMIT_PERC) + +enum mmtk_heap_mode { + RB_MMTK_DYNAMIC_HEAP, + RB_MMTK_FIXED_HEAP +}; + +MMTk_Builder * +rb_mmtk_builder_init(void) +{ + MMTk_Builder *builder = mmtk_builder_default(); + return builder; +} + +void * +rb_gc_impl_objspace_alloc(void) +{ + MMTk_Builder *builder = rb_mmtk_builder_init(); + MMTk_RubyBindingOptions binding_options = { + .suffix_size = RB_GC_OBJ_SUFFIX_SIZE, + }; + mmtk_init_binding(builder, &binding_options, &ruby_upcalls); + + return calloc(1, sizeof(struct objspace)); +} + +static void gc_run_finalizers(void *data); + +void +rb_gc_impl_objspace_init(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = true; + + objspace->finalizer_table = st_init_numtable(); + objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace); + + ccan_list_head_init(&objspace->ractor_caches); + + objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; + objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + + objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; +} + +void +rb_gc_impl_objspace_free(void *objspace_ptr) +{ + free(objspace_ptr); +} + +void * +rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor) +{ + struct objspace *objspace = objspace_ptr; + if (objspace->live_ractor_cache_count == 0) { + mmtk_initialize_collection(ractor); + } + objspace->live_ractor_cache_count++; + + struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache)); + ccan_list_add(&objspace->ractor_caches, &cache->list_node); + + cache->mutator = mmtk_bind_mutator(cache); + cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator); + + return cache; +} + +void +rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr) +{ + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *cache = cache_ptr; + + ccan_list_del(&cache->list_node); + + mmtk_flush_obj_free_buffer(cache); + + if (ruby_free_at_exit_p()) { + MMTK_ASSERT(objspace->live_ractor_cache_count > 0); + } + else { + MMTK_ASSERT(objspace->live_ractor_cache_count > 1); + } + + objspace->live_ractor_cache_count--; + + mmtk_destroy_mutator(cache->mutator); +} + +void rb_gc_impl_set_params(void *objspace_ptr) { } + +static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } + +#if SIZEOF_VALUE >= 8 +#define MMTK_HEAP_COUNT 12 +#define MMTK_MAX_OBJ_SIZE 1024 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 40, 64, 80, 96, 128, 160, 256, 512, 640, 768, MMTK_MAX_OBJ_SIZE, 0 +}; +#else +#define MMTK_HEAP_COUNT 5 +#define MMTK_MAX_OBJ_SIZE 512 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 +}; +#endif + +void +rb_gc_impl_init(void) +{ + VALUE gc_constants = rb_hash_new(); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); + // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); + OBJ_FREEZE(gc_constants); + rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants); + + // no-ops for compatibility + rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0); + + rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1); + rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0); + rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); +} + +size_t * +rb_gc_impl_heap_sizes(void *objspace_ptr) +{ + return heap_sizes; +} + +int +rb_mmtk_obj_free_iter_wrapper(VALUE obj, void *data) +{ + struct objspace *objspace = data; + + if (!RB_TYPE_P(obj, T_NONE)) { + rb_gc_obj_free_vm_weak_references(obj); + rb_gc_obj_free(objspace, obj); + } + + return 0; +} + +// Shutdown +static void each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data); + +void +rb_gc_impl_shutdown_free_objects(void *objspace_ptr) +{ + mmtk_set_gc_enabled(false); + each_object(objspace_ptr, rb_mmtk_obj_free_iter_wrapper, objspace_ptr); + mmtk_set_gc_enabled(true); +} + +// GC +void +rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact) +{ + mmtk_handle_user_collection_request(rb_gc_get_ractor_newobj_cache(), true, full_mark); +} + +bool +rb_gc_impl_during_gc_p(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + return objspace->world_stopped; +} + +static void +rb_gc_impl_prepare_heap_i(MMTk_ObjectReference obj, void *d) +{ + rb_gc_prepare_heap_process_object((VALUE)obj); +} + +void +rb_gc_impl_prepare_heap(void *objspace_ptr) +{ + mmtk_enumerate_objects(rb_gc_impl_prepare_heap_i, NULL); +} + +void +rb_gc_impl_gc_enable(void *objspace_ptr) +{ + mmtk_set_gc_enabled(true); +} + +void +rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc) +{ + mmtk_set_gc_enabled(false); +} + +bool +rb_gc_impl_gc_enabled_p(void *objspace_ptr) +{ + return mmtk_gc_enabled_p(); +} + +void +rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->gc_stress = RTEST(flag); +} + +VALUE +rb_gc_impl_stress_get(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_stress ? Qtrue : Qfalse; +} + +VALUE +rb_gc_impl_config_get(void *objspace_ptr) +{ + VALUE hash = rb_hash_new(); + + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_worker_count")), RB_ULONG2NUM(mmtk_worker_count())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_plan")), rb_str_new_cstr((const char *)mmtk_plan())); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_mode")), rb_str_new_cstr((const char *)mmtk_heap_mode())); + size_t heap_min = mmtk_heap_min(); + if (heap_min > 0) rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_min")), RB_ULONG2NUM(heap_min)); + rb_hash_aset(hash, ID2SYM(rb_intern_const("mmtk_heap_max")), RB_ULONG2NUM(mmtk_heap_max())); + + return hash; +} + +void +rb_gc_impl_config_set(void *objspace_ptr, VALUE hash) +{ + // TODO +} + +struct rb_gc_vm_context * +rb_gc_impl_get_vm_context(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return &objspace->vm_context; +} + +// Object allocation + +static VALUE +rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align) +{ + MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer; + if (bump_pointer == NULL) return 0; + + uintptr_t cursor = bump_pointer->cursor; + + // Ensure cursor is aligned + size_t mask = align - 1; + cursor = (cursor + mask) & ~mask; + + cursor += size; + + if (cursor > bump_pointer->limit) { + return 0; + } + else { + VALUE obj = cursor - size; + bump_pointer->cursor = cursor; + return obj; + } +} + +static bool +obj_can_parallel_free_p(VALUE obj) +{ + switch (RB_BUILTIN_TYPE(obj)) { + case T_ARRAY: + case T_BIGNUM: + case T_COMPLEX: + case T_FLOAT: + case T_HASH: + case T_OBJECT: + case T_RATIONAL: + case T_REGEXP: + case T_STRING: + case T_STRUCT: + case T_SYMBOL: + return true; + default: + return false; + } +} + +static void +mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache) +{ + if (cache->obj_free_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + if (cache->obj_free_non_parallel_count > 0) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } +} + +static inline void +mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj) +{ + if (obj_can_parallel_free_p(obj)) { + cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf, + cache->obj_free_parallel_count, true); + cache->obj_free_parallel_count = 0; + } + } + else { + cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj; + if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) { + mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf, + cache->obj_free_non_parallel_count, false); + cache->obj_free_non_parallel_count = 0; + } + } +} + +VALUE +rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size) +{ +#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0 + struct objspace *objspace = objspace_ptr; + struct MMTk_ractor_cache *ractor_cache = cache_ptr; + + if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big"); + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (alloc_size == heap_sizes[i]) break; + if (alloc_size < heap_sizes[i]) { + alloc_size = heap_sizes[i]; + break; + } + } + + if (objspace->gc_stress) { + mmtk_handle_user_collection_request(ractor_cache, false, false); + } + + // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)][suffix (RB_GC_OBJ_SUFFIX_SIZE)] + alloc_size += sizeof(VALUE) + RB_GC_OBJ_SUFFIX_SIZE; + + VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN); + if (!alloc_obj) { + alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + } + + alloc_obj++; + alloc_obj[-1] = alloc_size - sizeof(VALUE) - RB_GC_OBJ_SUFFIX_SIZE; + alloc_obj[0] = flags; + alloc_obj[1] = klass; + + // TODO: implement fast path for mmtk_post_alloc + mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT); + + // TODO: only add when object needs obj_free to be called + mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj); + + objspace->total_allocated_objects++; + + return (VALUE)alloc_obj; +} + +size_t +rb_gc_impl_obj_slot_size(VALUE obj) +{ + return ((VALUE *)obj)[-1]; +} + +size_t +rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) +{ + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + if (size == heap_sizes[i]) return i; + if (size < heap_sizes[i]) return i; + } + + rb_bug("size too big"); +} + +bool +rb_gc_impl_size_allocatable_p(size_t size) +{ + return size <= MMTK_MAX_OBJ_SIZE; +} + +// Malloc +void * +rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system malloc + return malloc(size); +} + +void * +rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed) +{ + // TODO: don't use system calloc + return calloc(1, size); +} + +void * +rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed) +{ + // TODO: don't use system realloc + return realloc(ptr, new_size); +} + +void +rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size) +{ + // TODO: don't use system free + free(ptr); +} + +void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { } + +// Marking +static inline VALUE +rb_mmtk_call_object_closure(VALUE obj, bool pin) +{ + if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) { + enum { info_size = 256 }; + char obj_info_buf[info_size]; + rb_raw_obj_info(obj_info_buf, info_size, obj); + + char parent_obj_info_buf[info_size]; + rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object); + + rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf); + } + + return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function( + rb_mmtk_gc_thread_tls->object_closure.rust_closure, + rb_mmtk_gc_thread_tls->gc_context, + (MMTk_ObjectReference)obj, + pin + ); +} + +void +rb_gc_impl_mark(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, false); +} + +void +rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr) +{ + if (RB_SPECIAL_CONST_P(*ptr)) return; + + VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false); + if (new_obj != *ptr) { + *ptr = new_obj; + } +} + +void +rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj) +{ + if (RB_SPECIAL_CONST_P(obj)) return; + + rb_mmtk_call_object_closure(obj, true); +} + +void +rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj) +{ + if (rb_gc_impl_pointer_to_heap_p(objspace_ptr, (const void *)obj)) { + rb_gc_impl_mark_and_pin(objspace_ptr, obj); + } +} + +void +rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj) +{ + RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE); + mmtk_declare_weak_references((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj) +{ + return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj); +} + +// Compaction +void +rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj) +{ + mmtk_register_pinning_obj((MMTk_ObjectReference)obj); +} + +bool +rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false) != obj; +} + +VALUE +rb_gc_impl_location(void *objspace_ptr, VALUE obj) +{ + return rb_mmtk_call_object_closure(obj, false); +} + +// Write barriers +void +rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + if (SPECIAL_CONST_P(b)) return; + +#ifdef MMTK_DEBUG + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)a)) { + char buff[256]; + rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a)); + } + + if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)b)) { + char buff[256]; + rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b)); + } +#endif + + MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE); + MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a); +} + +void +rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) +{ + mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj); +} + +void +rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) +{ + struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache(); + + mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)obj); +} + +// Heap walking +static void +each_objects_i(MMTk_ObjectReference obj, void *d) +{ + rb_darray(VALUE) *objs = d; + + rb_darray_append(objs, (VALUE)obj); +} + +static void +each_object(struct objspace *objspace, int (*func)(VALUE, void *), void *data) +{ + rb_darray(VALUE) objs; + rb_darray_make(&objs, 0); + + mmtk_enumerate_objects(each_objects_i, &objs); + + VALUE *obj_ptr; + rb_darray_foreach(objs, i, obj_ptr) { + if (!mmtk_is_mmtk_object((MMTk_ObjectReference)*obj_ptr)) continue; + + if (func(*obj_ptr, data) != 0) { + break; + } + } + + rb_darray_free(objs); +} + +struct rb_gc_impl_each_objects_data { + int (*func)(void *, void *, size_t, void *); + void *data; +}; + +static int +rb_gc_impl_each_objects_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_objects_data *data = d; + + size_t slot_size = rb_gc_impl_obj_slot_size(obj); + + return data->func((void *)obj, (void *)(obj + slot_size), slot_size, data->data); +} + +void +rb_gc_impl_each_objects(void *objspace_ptr, int (*func)(void *, void *, size_t, void *), void *data) +{ + struct rb_gc_impl_each_objects_data each_objects_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_objects_i, &each_objects_data); +} + +struct rb_gc_impl_each_object_data { + void (*func)(VALUE, void *); + void *data; +}; + +static int +rb_gc_impl_each_object_i(VALUE obj, void *d) +{ + struct rb_gc_impl_each_object_data *data = d; + + data->func(obj, data->data); + + return 0; +} + +void +rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE, void *), void *data) +{ + struct rb_gc_impl_each_object_data each_object_data = { + .func = func, + .data = data + }; + + each_object(objspace_ptr, rb_gc_impl_each_object_i, &each_object_data); +} + +// Finalizers +static VALUE +gc_run_finalizers_get_final(long i, void *data) +{ + VALUE table = (VALUE)data; + + return RARRAY_AREF(table, i + 1); +} + +static void +gc_run_finalizers(void *data) +{ + struct objspace *objspace = data; + + rb_gc_set_pending_interrupt(); + + while (objspace->finalizer_jobs != NULL) { + struct MMTk_final_job *job = objspace->finalizer_jobs; + objspace->finalizer_jobs = job->next; + + switch (job->kind) { + case MMTK_FINAL_JOB_DFREE: + job->as.dfree.func(job->as.dfree.data); + break; + case MMTK_FINAL_JOB_FINALIZE: { + VALUE finalizer_array = job->as.finalize.finalizer_array; + + rb_gc_run_obj_finalizer( + RARRAY_AREF(finalizer_array, 0), + RARRAY_LEN(finalizer_array) - 1, + gc_run_finalizers_get_final, + (void *)finalizer_array + ); + + RB_GC_GUARD(finalizer_array); + break; + } + } + + xfree(job); + } + + rb_gc_unset_pending_interrupt(); +} + +void +rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data) +{ + if (dfree == NULL) return; + + struct objspace *objspace = objspace_ptr; + + struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job)); + job->kind = MMTK_FINAL_JOB_DFREE; + job->as.dfree.func = dfree; + job->as.dfree.data = data; + + struct MMTk_final_job *prev; + do { + job->next = objspace->finalizer_jobs; + prev = RUBY_ATOMIC_PTR_CAS(objspace->finalizer_jobs, job->next, job); + } while (prev != job->next); + + if (!ruby_free_at_exit_p()) { + rb_postponed_job_trigger(objspace->finalizer_postponed_job); + } +} + +VALUE +rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + RBASIC(obj)->flags |= FL_FINALIZE; + + int lev = RB_GC_VM_LOCK(); + + if (st_lookup(objspace->finalizer_table, obj, &data)) { + table = (VALUE)data; + + /* avoid duplicate block, table is usually small */ + { + long len = RARRAY_LEN(table); + long i; + + for (i = 0; i < len; i++) { + VALUE recv = RARRAY_AREF(table, i); + if (rb_equal(recv, block)) { + RB_GC_VM_UNLOCK(lev); + return recv; + } + } + } + + rb_ary_push(table, block); + } + else { + table = rb_ary_new3(2, rb_obj_id(obj), block); + rb_obj_hide(table); + st_add_direct(objspace->finalizer_table, obj, table); + } + + RB_GC_VM_UNLOCK(lev); + + return block; +} + +void +rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + + st_data_t data = obj; + + int lev = RB_GC_VM_LOCK(); + st_delete(objspace->finalizer_table, &data, 0); + RB_GC_VM_UNLOCK(lev); + + FL_UNSET(obj, FL_FINALIZE); +} + +void +rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj) +{ + struct objspace *objspace = objspace_ptr; + VALUE table; + st_data_t data; + + if (!FL_TEST(obj, FL_FINALIZE)) return; + + int lev = RB_GC_VM_LOCK(); + if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) { + table = rb_ary_dup((VALUE)data); + RARRAY_ASET(table, 0, rb_obj_id(dest)); + st_insert(objspace->finalizer_table, dest, table); + FL_SET(dest, FL_FINALIZE); + } + else { + rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj)); + } + RB_GC_VM_UNLOCK(lev); +} + +static int +move_finalizer_from_table_i(st_data_t key, st_data_t val, st_data_t arg) +{ + struct objspace *objspace = (struct objspace *)arg; + + make_final_job(objspace, (VALUE)key, (VALUE)val); + + return ST_DELETE; +} + +void +rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + while (objspace->finalizer_table->num_entries) { + st_foreach(objspace->finalizer_table, move_finalizer_from_table_i, (st_data_t)objspace); + + gc_run_finalizers(objspace); + } + + unsigned int lev = RB_GC_VM_LOCK(); + { + struct MMTk_ractor_cache *rc; + ccan_list_for_each(&objspace->ractor_caches, rc, list_node) { + mmtk_flush_obj_free_buffer(rc); + } + + struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates(); + for (size_t i = 0; i < registered_candidates.len; i++) { + VALUE obj = (VALUE)registered_candidates.ptr[i]; + + if (rb_gc_shutdown_call_finalizer_p(obj)) { + rb_gc_obj_free(objspace_ptr, obj); + RBASIC(obj)->flags = 0; + } + } + mmtk_free_raw_vec_of_obj_ref(registered_candidates); + } + RB_GC_VM_UNLOCK(lev); + + gc_run_finalizers(objspace); +} + +// Forking + +void +rb_gc_impl_before_fork(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + retry: + objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); + + /* At this point, we know that all the Ractors are paused because of the + * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point, + * one or more Ractors could be paused there. However, mmtk_before_fork is + * not compatible with that because it assumes that the MMTk workers are idle, + * but the workers are not idle because they are busy working on a GC. + * + * This essentially implements a trylock. It will optimistically lock but will + * release the lock if it detects that any other Ractors are waiting in + * rb_mmtk_block_for_gc. + */ + rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count); + if (mutator_blocking_count != 0) { + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); + goto retry; + } + + mmtk_before_fork(); +} + +void +rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) +{ + struct objspace *objspace = objspace_ptr; + + mmtk_after_fork(rb_gc_get_ractor_newobj_cache()); + + RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev); +} + +// Statistics + +void +rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag) +{ + struct objspace *objspace = objspace_ptr; + + objspace->measure_gc_time = RTEST(flag); +} + +bool +rb_gc_impl_get_measure_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->measure_gc_time; +} + +unsigned long long +rb_gc_impl_get_total_time(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->total_gc_time; +} + +size_t +rb_gc_impl_gc_count(void *objspace_ptr) +{ + struct objspace *objspace = objspace_ptr; + + return objspace->gc_count; +} + +VALUE +rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key) +{ + VALUE hash = Qnil, key = Qnil; + + if (SYMBOL_P(hash_or_key)) { + key = hash_or_key; + } + else if (RB_TYPE_P(hash_or_key, T_HASH)) { + hash = hash_or_key; + } + else { + rb_bug("gc_info_decode: non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == ID2SYM(rb_intern_const(#name))) \ + return (attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, ID2SYM(rb_intern_const(#name)), (attr)); + + /* Hack to get StackProf working because it calls rb_gc_latest_gc_info with + * the :state key and expects a result. This always returns the :none state. */ + SET(state, ID2SYM(rb_intern_const("none"))); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +enum gc_stat_sym { + gc_stat_sym_count, + gc_stat_sym_moving_gc_count, + gc_stat_sym_time, + gc_stat_sym_total_allocated_objects, + gc_stat_sym_total_bytes, + gc_stat_sym_used_bytes, + gc_stat_sym_free_bytes, + gc_stat_sym_starting_heap_address, + gc_stat_sym_last_heap_address, + gc_stat_sym_weak_references_count, + gc_stat_sym_last +}; + +static VALUE gc_stat_symbols[gc_stat_sym_last]; + +static void +setup_gc_stat_symbols(void) +{ + if (gc_stat_symbols[0] == 0) { +#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) + S(count); + S(moving_gc_count); + S(time); + S(total_allocated_objects); + S(total_bytes); + S(used_bytes); + S(free_bytes); + S(starting_heap_address); + S(last_heap_address); + S(weak_references_count); + } +} + +VALUE +rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym) +{ + struct objspace *objspace = objspace_ptr; + VALUE hash = Qnil, key = Qnil; + + setup_gc_stat_symbols(); + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + hash = hash_or_sym; + } + else if (SYMBOL_P(hash_or_sym)) { + key = hash_or_sym; + } + else { + rb_bug("non-hash or symbol given"); + } + +#define SET(name, attr) \ + if (key == gc_stat_symbols[gc_stat_sym_##name]) \ + return SIZET2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr)); + + SET(count, objspace->gc_count); + SET(moving_gc_count, objspace->moving_gc_count); + SET(time, objspace->total_gc_time / (1000 * 1000)); + SET(total_allocated_objects, objspace->total_allocated_objects); + SET(total_bytes, mmtk_total_bytes()); + SET(used_bytes, mmtk_used_bytes()); + SET(free_bytes, mmtk_free_bytes()); + SET(starting_heap_address, (size_t)mmtk_starting_heap_address()); + SET(last_heap_address, (size_t)mmtk_last_heap_address()); + SET(weak_references_count, mmtk_weak_references_count()); +#undef SET + + if (!NIL_P(key)) { + // Matched key should return above + return Qundef; + } + + return hash; +} + +VALUE +rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) +{ + if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) { + int heap_idx = FIX2INT(heap_name); + if (heap_idx < 0 || heap_idx >= MMTK_HEAP_COUNT) { + rb_raise(rb_eArgError, "size pool index out of range"); + } + + if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) { + return SIZET2NUM(heap_sizes[heap_idx]); + } + + return Qundef; + } + + if (RB_TYPE_P(hash_or_sym, T_HASH)) { + return hash_or_sym; + } + + return Qundef; +} + +// Miscellaneous + +#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 1 +static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1]; + +struct rb_gc_object_metadata_entry * +rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj) +{ + static ID ID_object_id; + + if (!ID_object_id) { +#define I(s) ID_##s = rb_intern(#s); + I(object_id); +#undef I + } + + size_t n = 0; + +#define SET_ENTRY(na, v) do { \ + MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \ + object_metadata_entries[n].name = ID_##na; \ + object_metadata_entries[n].val = v; \ + n++; \ +} while (0) + + if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj)); + + object_metadata_entries[n].name = 0; + object_metadata_entries[n].val = 0; + + return object_metadata_entries; +} + +bool +rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr) +{ + if (ptr == NULL) return false; + if ((uintptr_t)ptr % sizeof(void*) != 0) return false; + return mmtk_is_mmtk_object((MMTk_Address)ptr); +} + +bool +rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj) +{ + return false; +} + +void rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event) { } + +void +rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj) +{ + if (mmtk_object_wb_unprotected_p((MMTk_ObjectReference)obj)) { + rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest); + } + + rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj); +} + +// GC Identification + +const char * +rb_gc_impl_active_gc_name(void) +{ + return "mmtk"; +} diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h new file mode 100644 index 0000000000..b11e2873e3 --- /dev/null +++ b/gc/mmtk/mmtk.h @@ -0,0 +1,175 @@ +#ifndef MMTK_H +#define MMTK_H + +/* Warning, this file is autogenerated by cbindgen from the mmtk-ruby repository. Don't modify this manually. */ + +#include <stdarg.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +typedef struct MMTk_Builder MMTk_Builder; +typedef struct MMTk_Mutator MMTk_Mutator; + +typedef struct MMTk_ractor_cache *MMTk_VMThread; +typedef struct MMTk_ractor_cache *MMTk_VMMutatorThread; +typedef struct MMTk_GCThreadTLS *MMTk_VMWorkerThread; +typedef void *MMTk_Address; +typedef void *MMTk_ObjectReference; +typedef void *MMTk_NullableObjectReference; +typedef uint32_t MMTk_AllocationSemantics; + +typedef struct MMTk_BumpPointer { + uintptr_t cursor; + uintptr_t limit; +} MMTk_BumpPointer; + + +#define MMTk_OBJREF_OFFSET 8 + +#define MMTk_MIN_OBJ_ALIGN 8 + +#define MMTk_GC_THREAD_KIND_WORKER 1 + +typedef struct MMTk_RubyBindingOptions { + size_t suffix_size; +} MMTk_RubyBindingOptions; + +typedef MMTk_ObjectReference (*MMTk_ObjectClosureFunction)(void*, void*, MMTk_ObjectReference, bool); + +typedef struct MMTk_ObjectClosure { + /** + * The function to be called from C. + */ + MMTk_ObjectClosureFunction c_function; + /** + * The pointer to the Rust-level closure object. + */ + void *rust_closure; +} MMTk_ObjectClosure; + +typedef struct MMTk_GCThreadTLS { + int kind; + void *gc_context; + struct MMTk_ObjectClosure object_closure; +} MMTk_GCThreadTLS; + +typedef struct MMTk_RubyUpcalls { + void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls); + bool (*is_mutator)(void); + void (*stop_the_world)(void); + void (*resume_mutators)(bool gc_may_move); + void (*block_for_gc)(MMTk_VMMutatorThread tls); + void (*before_updating_jit_code)(void); + void (*after_updating_jit_code)(void); + size_t (*number_of_mutators)(void); + void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data); + void (*scan_gc_roots)(void); + void (*scan_objspace)(void); + void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to); + void (*update_object_references)(MMTk_ObjectReference object); + void (*call_gc_mark_children)(MMTk_ObjectReference object); + void (*handle_weak_references)(MMTk_ObjectReference object, bool moving); + void (*call_obj_free)(MMTk_ObjectReference object); + size_t (*vm_live_bytes)(void); + void (*update_global_tables)(int tbl_idx, bool moving); + int (*global_tables_count)(void); + void (*update_finalizer_table)(void); + bool (*special_const_p)(MMTk_ObjectReference object); + void (*mutator_thread_panic_handler)(void); + void (*gc_thread_panic_handler)(void); +} MMTk_RubyUpcalls; + +typedef struct MMTk_RawVecOfObjRef { + MMTk_ObjectReference *ptr; + size_t len; + size_t capa; +} MMTk_RawVecOfObjRef; + +bool mmtk_is_live_object(MMTk_ObjectReference object); + +bool mmtk_is_reachable(MMTk_ObjectReference object); + +MMTk_Builder *mmtk_builder_default(void); + +void mmtk_init_binding(MMTk_Builder *builder, + const struct MMTk_RubyBindingOptions *binding_options, + const struct MMTk_RubyUpcalls *upcalls); + +void mmtk_initialize_collection(MMTk_VMThread tls); + +MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls); + +MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m); + +void mmtk_destroy_mutator(MMTk_Mutator *mutator); + +void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive); + +void mmtk_set_gc_enabled(bool enable); + +bool mmtk_gc_enabled_p(void); + +MMTk_Address mmtk_alloc(MMTk_Mutator *mutator, + size_t size, + size_t align, + size_t offset, + MMTk_AllocationSemantics semantics); + +void mmtk_post_alloc(MMTk_Mutator *mutator, + MMTk_ObjectReference refer, + size_t bytes, + MMTk_AllocationSemantics semantics); + +void mmtk_add_obj_free_candidates(const MMTk_ObjectReference *objects, + size_t count, + bool can_parallel_free); + +void mmtk_declare_weak_references(MMTk_ObjectReference object); + +bool mmtk_weak_references_alive_p(MMTk_ObjectReference object); + +size_t mmtk_weak_references_count(void); + +void mmtk_register_pinning_obj(MMTk_ObjectReference obj); + +void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object); + +void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object); + +bool mmtk_object_wb_unprotected_p(MMTk_ObjectReference object); + +void mmtk_enumerate_objects(void (*callback)(MMTk_ObjectReference, void*), void *data); + +struct MMTk_RawVecOfObjRef mmtk_get_all_obj_free_candidates(void); + +void mmtk_free_raw_vec_of_obj_ref(struct MMTk_RawVecOfObjRef raw_vec); + +void mmtk_before_fork(void); + +void mmtk_after_fork(MMTk_VMThread tls); + +size_t mmtk_total_bytes(void); + +size_t mmtk_used_bytes(void); + +size_t mmtk_free_bytes(void); + +MMTk_Address mmtk_starting_heap_address(void); + +MMTk_Address mmtk_last_heap_address(void); + +size_t mmtk_worker_count(void); + +const uint8_t *mmtk_plan(void); + +const uint8_t *mmtk_heap_mode(void); + +size_t mmtk_heap_min(void); + +size_t mmtk_heap_max(void); + +bool mmtk_is_mmtk_object(MMTk_Address addr); + +#endif /* MMTK_H */ diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs new file mode 100644 index 0000000000..30890e0853 --- /dev/null +++ b/gc/mmtk/src/abi.rs @@ -0,0 +1,335 @@ +use crate::api::RubyMutator; +use crate::extra_assert; +use crate::Ruby; +use libc::c_int; +use mmtk::scheduler::GCWorker; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMWorkerThread; + +// For the C binding +pub const OBJREF_OFFSET: usize = 8; +pub const MIN_OBJ_ALIGN: usize = 8; // Even on 32-bit machine. A Ruby object is at least 40 bytes large. + +pub const GC_THREAD_KIND_WORKER: libc::c_int = 1; + +const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF; + +// An opaque type for the C counterpart. +#[allow(non_camel_case_types)] +pub struct st_table; + +#[repr(C)] +pub struct HiddenHeader { + pub prefix: usize, +} + +impl HiddenHeader { + #[inline(always)] + pub fn is_sane(&self) -> bool { + self.prefix & !HIDDEN_SIZE_MASK == 0 + } + + #[inline(always)] + fn assert_sane(&self) { + extra_assert!( + self.is_sane(), + "Hidden header is corrupted: {:x}", + self.prefix + ); + } + + pub fn payload_size(&self) -> usize { + self.assert_sane(); + self.prefix & HIDDEN_SIZE_MASK + } +} + +/// Provide convenient methods for accessing Ruby objects. +/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods. +pub struct RubyObjectAccess { + objref: ObjectReference, +} + +impl RubyObjectAccess { + pub fn from_objref(objref: ObjectReference) -> Self { + Self { objref } + } + + pub fn obj_start(&self) -> Address { + self.objref.to_raw_address().sub(Self::prefix_size()) + } + + pub fn payload_addr(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn suffix_addr(&self) -> Address { + self.objref.to_raw_address().add(self.payload_size()) + } + + pub fn obj_end(&self) -> Address { + self.suffix_addr() + Self::suffix_size() + } + + fn hidden_header(&self) -> &'static HiddenHeader { + unsafe { self.obj_start().as_ref() } + } + + #[allow(unused)] // Maybe we need to mutate the hidden header in the future. + fn hidden_header_mut(&self) -> &'static mut HiddenHeader { + unsafe { self.obj_start().as_mut_ref() } + } + + pub fn payload_size(&self) -> usize { + self.hidden_header().payload_size() + } + + fn flags_field(&self) -> Address { + self.objref.to_raw_address() + } + + pub fn load_flags(&self) -> usize { + unsafe { self.flags_field().load::<usize>() } + } + + pub fn prefix_size() -> usize { + // Currently, a hidden size field of word size is placed before each object. + OBJREF_OFFSET + } + + pub fn suffix_size() -> usize { + // In RACTOR_CHECK_MODE, Ruby hides a field after each object to hold the Ractor ID. + unsafe { crate::BINDING_FAST.suffix_size } + } + + pub fn object_size(&self) -> usize { + Self::prefix_size() + self.payload_size() + Self::suffix_size() + } +} + +type ObjectClosureFunction = + extern "C" fn(*mut libc::c_void, *mut libc::c_void, ObjectReference, bool) -> ObjectReference; + +#[repr(C)] +pub struct ObjectClosure { + /// The function to be called from C. + pub c_function: ObjectClosureFunction, + /// The pointer to the Rust-level closure object. + pub rust_closure: *mut libc::c_void, +} + +impl Default for ObjectClosure { + fn default() -> Self { + Self { + c_function: THE_UNREGISTERED_CLOSURE_FUNC, + rust_closure: std::ptr::null_mut(), + } + } +} + +/// Rust doesn't require function items to have a unique address. +/// We therefore force using this particular constant. +/// +/// See: https://rust-lang.github.io/rust-clippy/master/index.html#fn_address_comparisons +const THE_UNREGISTERED_CLOSURE_FUNC: ObjectClosureFunction = ObjectClosure::c_function_unregistered; + +impl ObjectClosure { + /// Set this ObjectClosure temporarily to `visit_object`, and execute `f`. During the execution of + /// `f`, the Ruby VM may call this ObjectClosure. When the Ruby VM calls this ObjectClosure, + /// it effectively calls `visit_object`. + /// + /// This method is intended to run Ruby VM code in `f` with temporarily modified behavior of + /// `rb_gc_mark`, `rb_gc_mark_movable` and `rb_gc_location` + /// + /// Both `f` and `visit_object` may access and modify local variables in the environment where + /// `set_temporarily_and_run_code` called. + /// + /// Note that this function is not reentrant. Don't call this function in either `callback` or + /// `f`. + pub fn set_temporarily_and_run_code<'env, T, F1, F2>( + &mut self, + mut visit_object: F1, + f: F2, + ) -> T + where + F1: 'env + FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + F2: 'env + FnOnce() -> T, + { + debug_assert!( + std::ptr::fn_addr_eq(self.c_function, THE_UNREGISTERED_CLOSURE_FUNC), + "set_temporarily_and_run_code is recursively called." + ); + self.c_function = Self::c_function_registered::<F1>; + self.rust_closure = &mut visit_object as *mut F1 as *mut libc::c_void; + let result = f(); + *self = Default::default(); + result + } + + extern "C" fn c_function_registered<F>( + rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference + where + F: FnMut(&'static mut GCWorker<Ruby>, ObjectReference, bool) -> ObjectReference, + { + let rust_closure = unsafe { &mut *(rust_closure as *mut F) }; + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + rust_closure(worker, object, pin) + } + + extern "C" fn c_function_unregistered( + _rust_closure: *mut libc::c_void, + worker: *mut libc::c_void, + object: ObjectReference, + pin: bool, + ) -> ObjectReference { + let worker = unsafe { &mut *(worker as *mut GCWorker<Ruby>) }; + panic!( + "object_closure is not set. worker ordinal: {}, object: {}, pin: {}", + worker.ordinal, object, pin + ); + } +} + +#[repr(C)] +pub struct GCThreadTLS { + pub kind: libc::c_int, + pub gc_context: *mut libc::c_void, + pub object_closure: ObjectClosure, +} + +impl GCThreadTLS { + fn new(kind: libc::c_int, gc_context: *mut libc::c_void) -> Self { + Self { + kind, + gc_context, + object_closure: Default::default(), + } + } + + pub fn for_worker(gc_context: *mut GCWorker<Ruby>) -> Self { + Self::new(GC_THREAD_KIND_WORKER, gc_context as *mut libc::c_void) + } + + pub fn from_vwt(vwt: VMWorkerThread) -> *mut GCThreadTLS { + unsafe { std::mem::transmute(vwt) } + } + + /// Cast a pointer to `GCThreadTLS` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn check_cast(ptr: *mut GCThreadTLS) -> &'static mut GCThreadTLS { + assert!(!ptr.is_null()); + let result = unsafe { &mut *ptr }; + debug_assert!({ + let kind = result.kind; + kind == GC_THREAD_KIND_WORKER + }); + result + } + + /// Cast a pointer to `VMWorkerThread` to a ref, with assertion for null pointer. + /// + /// # Safety + /// + /// Has undefined behavior if `ptr` is invalid. + pub unsafe fn from_vwt_check(vwt: VMWorkerThread) -> &'static mut GCThreadTLS { + let ptr = Self::from_vwt(vwt); + unsafe { Self::check_cast(ptr) } + } + + #[allow(clippy::not_unsafe_ptr_arg_deref)] // `transmute` does not dereference pointer + pub fn to_vwt(ptr: *mut Self) -> VMWorkerThread { + unsafe { std::mem::transmute(ptr) } + } + + pub fn worker<'w>(&mut self) -> &'w mut GCWorker<Ruby> { + // NOTE: The returned ref points to the worker which does not have the same lifetime as self. + assert!(self.kind == GC_THREAD_KIND_WORKER); + unsafe { &mut *(self.gc_context as *mut GCWorker<Ruby>) } + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RawVecOfObjRef { + pub ptr: *mut ObjectReference, + pub len: usize, + pub capa: usize, +} + +impl RawVecOfObjRef { + pub fn from_vec(vec: Vec<ObjectReference>) -> RawVecOfObjRef { + // Note: Vec::into_raw_parts is unstable. We implement it manually. + let mut vec = std::mem::ManuallyDrop::new(vec); + let (ptr, len, capa) = (vec.as_mut_ptr(), vec.len(), vec.capacity()); + + RawVecOfObjRef { ptr, len, capa } + } + + /// # Safety + /// + /// This function turns raw pointer into a Vec without check. + pub unsafe fn into_vec(self) -> Vec<ObjectReference> { + unsafe { Vec::from_raw_parts(self.ptr, self.len, self.capa) } + } +} + +impl From<Vec<ObjectReference>> for RawVecOfObjRef { + fn from(v: Vec<ObjectReference>) -> Self { + Self::from_vec(v) + } +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyBindingOptions { + pub suffix_size: usize, +} + +#[repr(C)] +#[derive(Clone)] +pub struct RubyUpcalls { + pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS), + pub is_mutator: extern "C" fn() -> bool, + pub stop_the_world: extern "C" fn(), + pub resume_mutators: extern "C" fn(gc_may_move: bool), + pub block_for_gc: extern "C" fn(tls: VMMutatorThread), + pub before_updating_jit_code: extern "C" fn(), + pub after_updating_jit_code: extern "C" fn(), + pub number_of_mutators: extern "C" fn() -> usize, + pub get_mutators: extern "C" fn( + visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void), + data: *mut libc::c_void, + ), + pub scan_gc_roots: extern "C" fn(), + pub scan_objspace: extern "C" fn(), + pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference), + pub update_object_references: extern "C" fn(object: ObjectReference), + pub call_gc_mark_children: extern "C" fn(object: ObjectReference), + pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool), + pub call_obj_free: extern "C" fn(object: ObjectReference), + pub vm_live_bytes: extern "C" fn() -> usize, + pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool), + pub global_tables_count: extern "C" fn() -> c_int, + pub update_finalizer_table: extern "C" fn(), + pub special_const_p: extern "C" fn(object: ObjectReference) -> bool, + pub mutator_thread_panic_handler: extern "C" fn(), + pub gc_thread_panic_handler: extern "C" fn(), +} + +unsafe impl Sync for RubyUpcalls {} + +#[repr(C)] +#[derive(Clone)] +pub struct HeapBounds { + pub start: *mut libc::c_void, + pub end: *mut libc::c_void, +} diff --git a/gc/mmtk/src/active_plan.rs b/gc/mmtk/src/active_plan.rs new file mode 100644 index 0000000000..80372a7576 --- /dev/null +++ b/gc/mmtk/src/active_plan.rs @@ -0,0 +1,56 @@ +use std::collections::VecDeque; +use std::marker::PhantomData; + +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::util::opaque_pointer::*; +use mmtk::vm::ActivePlan; +use mmtk::Mutator; + +pub struct VMActivePlan {} + +impl ActivePlan<Ruby> for VMActivePlan { + fn number_of_mutators() -> usize { + (upcalls().number_of_mutators)() + } + + fn is_mutator(_tls: VMThread) -> bool { + (upcalls().is_mutator)() + } + + fn mutator(_tls: VMMutatorThread) -> &'static mut Mutator<Ruby> { + unimplemented!() + } + + fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut Mutator<Ruby>> + 'a> { + let mut mutators = VecDeque::new(); + (upcalls().get_mutators)( + add_mutator_to_vec, + &mut mutators as *mut VecDeque<&mut Mutator<Ruby>> as _, + ); + + Box::new(RubyMutatorIterator { + mutators, + phantom_data: PhantomData, + }) + } +} + +extern "C" fn add_mutator_to_vec(mutator: *mut Mutator<Ruby>, mutators: *mut libc::c_void) { + let mutators = unsafe { &mut *(mutators as *mut VecDeque<*mut Mutator<Ruby>>) }; + mutators.push_back(unsafe { &mut *mutator }); +} + +struct RubyMutatorIterator<'a> { + mutators: VecDeque<&'a mut Mutator<Ruby>>, + phantom_data: PhantomData<&'a ()>, +} + +impl<'a> Iterator for RubyMutatorIterator<'a> { + type Item = &'a mut Mutator<Ruby>; + + fn next(&mut self) -> Option<Self::Item> { + self.mutators.pop_front() + } +} diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs new file mode 100644 index 0000000000..c0540fe0c8 --- /dev/null +++ b/gc/mmtk/src/api.rs @@ -0,0 +1,551 @@ +// Functions in this module are unsafe for one reason: +// They are called by C functions and they need to pass raw pointers to Rust. +#![allow(clippy::missing_safety_doc)] + +use mmtk::util::alloc::BumpPointer; +use mmtk::util::alloc::ImmixAllocator; +use mmtk::util::conversions; +use mmtk::util::options::PlanSelector; +use std::str::FromStr; +use std::sync::atomic::Ordering; + +use crate::abi::RawVecOfObjRef; +use crate::abi::RubyBindingOptions; +use crate::abi::RubyUpcalls; +use crate::binding; +use crate::binding::RubyBinding; +use crate::heap::CpuHeapTriggerConfig; +use crate::heap::RubyHeapTriggerConfig; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::heap::RUBY_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::utils::default_heap_max; +use crate::utils::parse_capacity; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::memory_manager::mmtk_init; +use mmtk::util::constants::MIN_OBJECT_SIZE; +use mmtk::util::options::GCTriggerSelector; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::AllocationSemantics; +use mmtk::MMTKBuilder; +use mmtk::Mutator; + +pub type RubyMutator = Mutator<Ruby>; + +#[no_mangle] +pub extern "C" fn mmtk_is_live_object(object: ObjectReference) -> bool { + memory_manager::is_live_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool { + object.is_reachable() +} + +// =============== Bootup =============== + +fn parse_env_var_with<T, F: FnOnce(&str) -> Option<T>>(key: &str, parse: F) -> Option<T> { + let val = match std::env::var(key) { + Ok(val) => val, + Err(std::env::VarError::NotPresent) => return None, + Err(std::env::VarError::NotUnicode(os_string)) => { + eprintln!("[FATAL] Invalid {key} {os_string:?}"); + std::process::exit(1); + } + }; + + let parsed = parse(&val).unwrap_or_else(|| { + eprintln!("[FATAL] Invalid {key} {val}"); + std::process::exit(1); + }); + + Some(parsed) +} + +fn parse_env_var<T: FromStr>(key: &str) -> Option<T> { + parse_env_var_with(key, |s| s.parse().ok()) +} + +fn mmtk_builder_default_parse_threads() -> Option<usize> { + parse_env_var("MMTK_THREADS") +} + +fn mmtk_builder_default_parse_heap_min() -> usize { + const DEFAULT_HEAP_MIN: usize = 1 << 20; + parse_env_var_with("MMTK_HEAP_MIN", parse_capacity).unwrap_or(DEFAULT_HEAP_MIN) +} + +fn mmtk_builder_default_parse_heap_max() -> usize { + parse_env_var_with("MMTK_HEAP_MAX", parse_capacity).unwrap_or_else(default_heap_max) +} + +fn parse_float_env_var(key: &str, default: f64, min: f64, max: f64) -> f64 { + parse_env_var_with(key, |s| { + let mut float = f64::from_str(s).unwrap_or(default); + + if float <= min { + eprintln!( + "{key} has value {float} which must be greater than {min}, using default instead" + ); + float = default; + } + + if float >= max { + eprintln!( + "{key} has value {float} which must be less than {max}, using default instead" + ); + float = default; + } + + Some(float) + }) + .unwrap_or(default) +} + +fn mmtk_builder_default_parse_heap_mode(heap_min: usize, heap_max: usize) -> GCTriggerSelector { + let make_fixed = || GCTriggerSelector::FixedHeapSize(heap_max); + let make_dynamic = || GCTriggerSelector::DynamicHeapSize(heap_min, heap_max); + + parse_env_var_with("MMTK_HEAP_MODE", |s| match s { + "fixed" => Some(make_fixed()), + "dynamic" => Some(make_dynamic()), + "ruby" => { + let min_ratio = parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", 0.2, 0.0, 1.0); + let goal_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", 0.4, min_ratio, 1.0); + let max_ratio = + parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", 0.65, goal_ratio, 1.0); + + crate::heap::RUBY_HEAP_TRIGGER_CONFIG + .set(RubyHeapTriggerConfig { + min_heap_pages: conversions::bytes_to_pages_up(heap_min), + max_heap_pages: conversions::bytes_to_pages_up(heap_max), + heap_pages_min_ratio: min_ratio, + heap_pages_goal_ratio: goal_ratio, + heap_pages_max_ratio: max_ratio, + }) + .unwrap_or_else(|_| panic!("RUBY_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + "cpu" => { + // CPU-overhead-driven heap sizing based on Tavakolisomeh et al., + // "Heap Size Adjustment with CPU Control", MPLR '23. + // + // Target is expressed as a percentage (0, 100) via + // `MMTK_GC_CPU_TARGET`. The paper recommends 15 for ZGC (a + // concurrent collector); we default to 5 for MMTk-Ruby. With + // MMTk's stop-the-world Immix, every percent of GC CPU is also + // a percent of wall-clock the mutator is blocked on, so a much + // smaller budget is appropriate. An empirical sweep across + // ruby-bench (railsbench, lobsters, psych-load, liquid-render, + // lee) found target=5 to be Pareto-optimal: ~6% geomean speedup + // vs. the `ruby` heap mode with effectively identical geomean + // peak RSS. + let target_percent = parse_float_env_var("MMTK_GC_CPU_TARGET", 5.0, 0.0, 100.0); + let window_size = parse_env_var::<usize>("MMTK_GC_CPU_WINDOW").unwrap_or(3); + let window_size = window_size.max(1); + + let min_heap_pages = conversions::bytes_to_pages_up(heap_min); + let max_heap_pages = conversions::bytes_to_pages_up(heap_max); + // Start at the min heap size, as the other delegated triggers do. + // The control loop will adjust from here after the first GC cycle. + let initial_heap_pages = min_heap_pages; + + CPU_HEAP_TRIGGER_CONFIG + .set(CpuHeapTriggerConfig { + min_heap_pages, + max_heap_pages, + initial_heap_pages, + target_gc_cpu: target_percent / 100.0, + window_size, + }) + .unwrap_or_else(|_| panic!("CPU_HEAP_TRIGGER_CONFIG is already set")); + + Some(GCTriggerSelector::Delegated) + } + _ => None, + }) + .unwrap_or_else(make_dynamic) +} + +fn mmtk_builder_default_parse_plan() -> PlanSelector { + parse_env_var_with("MMTK_PLAN", |s| match s { + "NoGC" => Some(PlanSelector::NoGC), + "MarkSweep" => Some(PlanSelector::MarkSweep), + "Immix" => Some(PlanSelector::Immix), + _ => None, + }) + .unwrap_or(PlanSelector::Immix) +} + +#[no_mangle] +pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder { + let mut builder = MMTKBuilder::new_no_env_vars(); + builder.options.no_finalizer.set(true); + + if let Some(threads) = mmtk_builder_default_parse_threads() { + if !builder.options.threads.set(threads) { + // MMTk will validate it and reject 0. + eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}"); + std::process::exit(1); + } + } + + let heap_min = mmtk_builder_default_parse_heap_min(); + + let heap_max = mmtk_builder_default_parse_heap_max(); + + if heap_min >= heap_max { + eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})"); + std::process::exit(1); + } + + builder + .options + .gc_trigger + .set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max)); + + builder.options.plan.set(mmtk_builder_default_parse_plan()); + + Box::into_raw(Box::new(builder)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_init_binding( + builder: *mut MMTKBuilder, + binding_options: *const RubyBindingOptions, + upcalls: *const RubyUpcalls, +) { + crate::MUTATOR_THREAD_PANIC_HANDLER + .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler) + .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized")); + + crate::set_panic_hook(); + + let builder: Box<MMTKBuilder> = unsafe { Box::from_raw(builder) }; + let binding_options = unsafe { (*binding_options).clone() }; + let mmtk_boxed = mmtk_init(&builder); + let mmtk_static = Box::leak(Box::new(mmtk_boxed)); + + let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls); + binding + .weak_proc + .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk)); + + crate::BINDING + .set(binding) + .unwrap_or_else(|_| panic!("Binding is already initialized")); +} + +#[no_mangle] +pub extern "C" fn mmtk_initialize_collection(tls: VMThread) { + memory_manager::initialize_collection(mmtk(), tls) +} + +#[no_mangle] +pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator { + Box::into_raw(memory_manager::bind_mutator(mmtk(), tls)) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer { + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::Immix => { + let mutator: &mut Mutator<Ruby> = unsafe { &mut *m }; + let allocator = + unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) }; + + if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() { + &mut immix_allocator.bump_pointer as *mut BumpPointer + } else { + panic!("Failed to get bump pointer allocator"); + } + } + _ => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) { + // notify mmtk-core about destroyed mutator + memory_manager::destroy_mutator(unsafe { &mut *mutator }); + // turn the ptr back to a box, and let Rust properly reclaim it + let _ = unsafe { Box::from_raw(mutator) }; +} + +// =============== GC =============== + +#[no_mangle] +pub extern "C" fn mmtk_handle_user_collection_request( + tls: VMMutatorThread, + force: bool, + exhaustive: bool, +) { + crate::mmtk().handle_user_collection_request(tls, force, exhaustive); +} + +#[no_mangle] +pub extern "C" fn mmtk_set_gc_enabled(enable: bool) { + crate::CONFIGURATION + .gc_enabled + .store(enable, Ordering::Relaxed); +} + +#[no_mangle] +pub extern "C" fn mmtk_gc_enabled_p() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) +} + +// =============== Object allocation =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_alloc( + mutator: *mut RubyMutator, + size: usize, + align: usize, + offset: usize, + semantics: AllocationSemantics, +) -> Address { + let clamped_size = size.max(MIN_OBJECT_SIZE); + memory_manager::alloc::<Ruby>( + unsafe { &mut *mutator }, + clamped_size, + align, + offset, + semantics, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_post_alloc( + mutator: *mut RubyMutator, + refer: ObjectReference, + bytes: usize, + semantics: AllocationSemantics, +) { + memory_manager::post_alloc::<Ruby>(unsafe { &mut *mutator }, refer, bytes, semantics) +} + +#[no_mangle] +pub unsafe extern "C" fn mmtk_add_obj_free_candidates( + objects: *const ObjectReference, + count: usize, + can_parallel_free: bool, +) { + let objects = unsafe { std::slice::from_raw_parts(objects, count) }; + binding() + .weak_proc + .add_obj_free_candidates_batch(objects, can_parallel_free) +} + +// =============== Weak references =============== + +#[no_mangle] +pub extern "C" fn mmtk_declare_weak_references(object: ObjectReference) { + binding().weak_proc.add_weak_reference(object); +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool { + object.is_reachable() +} + +#[no_mangle] +pub extern "C" fn mmtk_weak_references_count() -> usize { + binding().weak_proc.weak_references_count() +} + +// =============== Compaction =============== + +#[no_mangle] +pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) { + crate::binding().pinning_registry.register(obj); +} + +// =============== Write barriers =============== + +#[no_mangle] +pub unsafe extern "C" fn mmtk_object_reference_write_post( + mutator: *mut RubyMutator, + object: ObjectReference, +) { + let ignored_slot = RubySlot::from_address(Address::ZERO); + let ignored_target = ObjectReference::from_raw_address(Address::ZERO); + mmtk::memory_manager::object_reference_write_post( + unsafe { &mut *mutator }, + object, + ignored_slot, + ignored_target, + ) +} + +#[no_mangle] +pub extern "C" fn mmtk_register_wb_unprotected_object(object: ObjectReference) { + crate::binding().register_wb_unprotected_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_object_wb_unprotected_p(object: ObjectReference) -> bool { + crate::binding().object_wb_unprotected_p(object) +} + +// =============== Heap walking =============== + +#[no_mangle] +pub extern "C" fn mmtk_enumerate_objects( + callback: extern "C" fn(ObjectReference, *mut libc::c_void), + data: *mut libc::c_void, +) { + crate::mmtk().enumerate_objects(|object| { + callback(object, data); + }) +} + +// =============== Finalizers =============== + +#[no_mangle] +pub extern "C" fn mmtk_get_all_obj_free_candidates() -> RawVecOfObjRef { + let vec = binding().weak_proc.get_all_obj_free_candidates(); + RawVecOfObjRef::from_vec(vec) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_raw_vec_of_obj_ref(raw_vec: RawVecOfObjRef) { + unsafe { raw_vec.into_vec() }; +} + +// =============== Forking =============== + +#[no_mangle] +pub extern "C" fn mmtk_before_fork() { + mmtk().prepare_to_fork(); + binding().join_all_gc_threads(); +} + +#[no_mangle] +pub extern "C" fn mmtk_after_fork(tls: VMThread) { + mmtk().after_fork(tls); +} + +// =============== Statistics =============== + +#[no_mangle] +pub extern "C" fn mmtk_total_bytes() -> usize { + memory_manager::total_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_used_bytes() -> usize { + memory_manager::used_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_free_bytes() -> usize { + memory_manager::free_bytes(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_starting_heap_address() -> Address { + memory_manager::starting_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_last_heap_address() -> Address { + memory_manager::last_heap_address() +} + +#[no_mangle] +pub extern "C" fn mmtk_worker_count() -> usize { + memory_manager::num_of_workers(mmtk()) +} + +#[no_mangle] +pub extern "C" fn mmtk_plan() -> *const u8 { + static NO_GC: &[u8] = b"NoGC\0"; + static MARK_SWEEP: &[u8] = b"MarkSweep\0"; + static IMMIX: &[u8] = b"Immix\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().plan { + PlanSelector::NoGC => NO_GC.as_ptr(), + PlanSelector::MarkSweep => MARK_SWEEP.as_ptr(), + PlanSelector::Immix => IMMIX.as_ptr(), + _ => panic!("Unknown plan"), + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_mode() -> *const u8 { + static FIXED_HEAP: &[u8] = b"fixed\0"; + static DYNAMIC_HEAP: &[u8] = b"dynamic\0"; + static RUBY_HEAP: &[u8] = b"ruby\0"; + static CPU_HEAP: &[u8] = b"cpu\0"; + + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => FIXED_HEAP.as_ptr(), + GCTriggerSelector::DynamicHeapSize(_, _) => DYNAMIC_HEAP.as_ptr(), + GCTriggerSelector::Delegated => { + // Two delegated triggers exist; disambiguate via the populated + // config singleton. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + CPU_HEAP.as_ptr() + } else { + RUBY_HEAP.as_ptr() + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_min() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(_) => 0, + GCTriggerSelector::DynamicHeapSize(min_size, _) => min_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.min_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .min_heap_pages, + ) + } + } + } +} + +#[no_mangle] +pub extern "C" fn mmtk_heap_max() -> usize { + match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger { + GCTriggerSelector::FixedHeapSize(max_size) => max_size, + GCTriggerSelector::DynamicHeapSize(_, max_size) => max_size, + GCTriggerSelector::Delegated => { + if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() { + conversions::pages_to_bytes(cfg.max_heap_pages) + } else { + conversions::pages_to_bytes( + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("RUBY_HEAP_TRIGGER_CONFIG not set") + .max_heap_pages, + ) + } + } + } +} + +// =============== Miscellaneous =============== + +#[no_mangle] +pub extern "C" fn mmtk_is_mmtk_object(addr: Address) -> bool { + debug_assert!(!addr.is_zero()); + debug_assert!(addr.is_aligned_to(mmtk::util::is_mmtk_object::VO_BIT_REGION_SIZE)); + memory_manager::is_mmtk_object(addr).is_some() +} diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs new file mode 100644 index 0000000000..36d4a992fd --- /dev/null +++ b/gc/mmtk/src/binding.rs @@ -0,0 +1,129 @@ +use std::collections::HashSet; +use std::ffi::CString; +use std::sync::atomic::AtomicBool; +use std::sync::Mutex; +use std::thread::JoinHandle; + +use mmtk::util::ObjectReference; +use mmtk::MMTK; + +use crate::abi; +use crate::abi::RubyBindingOptions; +use crate::pinning_registry::PinningRegistry; +use crate::weak_proc::WeakProcessor; +use crate::Ruby; + +pub struct RubyBindingFast { + pub suffix_size: usize, +} + +impl Default for RubyBindingFast { + fn default() -> Self { + Self::new() + } +} + +impl RubyBindingFast { + pub const fn new() -> Self { + Self { suffix_size: 0 } + } +} + +pub struct RubyConfiguration { + pub gc_enabled: AtomicBool, +} + +impl Default for RubyConfiguration { + fn default() -> Self { + Self::new() + } +} + +impl RubyConfiguration { + pub const fn new() -> Self { + Self { + // Mimic the old behavior when the gc_enabled flag was in mmtk-core. + // We may refactor it so that it is false by default. + gc_enabled: AtomicBool::new(true), + } + } +} + +pub struct RubyBinding { + pub mmtk: &'static MMTK<Ruby>, + pub options: RubyBindingOptions, + pub upcalls: *const abi::RubyUpcalls, + pub plan_name: Mutex<Option<CString>>, + pub weak_proc: WeakProcessor, + pub pinning_registry: PinningRegistry, + pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>, + pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>, +} + +unsafe impl Sync for RubyBinding {} +unsafe impl Send for RubyBinding {} + +impl RubyBinding { + pub fn new( + mmtk: &'static MMTK<Ruby>, + binding_options: &RubyBindingOptions, + upcalls: *const abi::RubyUpcalls, + ) -> Self { + unsafe { + crate::BINDING_FAST.suffix_size = binding_options.suffix_size; + } + + Self { + mmtk, + options: binding_options.clone(), + upcalls, + plan_name: Mutex::new(None), + weak_proc: WeakProcessor::new(), + pinning_registry: PinningRegistry::new(), + gc_thread_join_handles: Default::default(), + wb_unprotected_objects: Default::default(), + } + } + + pub fn upcalls(&self) -> &'static abi::RubyUpcalls { + unsafe { &*self.upcalls as &'static abi::RubyUpcalls } + } + + pub fn get_plan_name_c(&self) -> *const libc::c_char { + let mut plan_name = self.plan_name.lock().unwrap(); + if plan_name.is_none() { + let name_string = format!("{:?}", *self.mmtk.get_options().plan); + let c_string = CString::new(name_string) + .unwrap_or_else(|e| panic!("Failed converting plan name to CString: {e}")); + *plan_name = Some(c_string); + } + plan_name.as_deref().unwrap().as_ptr() + } + + pub fn join_all_gc_threads(&self) { + let handles = { + let mut guard = self.gc_thread_join_handles.lock().unwrap(); + std::mem::take(&mut *guard) + }; + + debug!("Joining GC threads..."); + let total = handles.len(); + let mut joined = 0; + for handle in handles { + handle.join().unwrap(); + joined += 1; + debug!("{joined}/{total} GC threads joined."); + } + } + + pub fn register_wb_unprotected_object(&self, object: ObjectReference) { + debug!("Registering WB-unprotected object: {object}"); + let mut objects = self.wb_unprotected_objects.lock().unwrap(); + objects.insert(object); + } + + pub fn object_wb_unprotected_p(&self, object: ObjectReference) -> bool { + let objects = self.wb_unprotected_objects.lock().unwrap(); + objects.contains(&object) + } +} diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs new file mode 100644 index 0000000000..648efa4e27 --- /dev/null +++ b/gc/mmtk/src/collection.rs @@ -0,0 +1,122 @@ +use crate::abi::GCThreadTLS; + +use crate::api::RubyMutator; +use crate::heap::CpuHeapTrigger; +use crate::heap::RubyHeapTrigger; +use crate::heap::CPU_HEAP_TRIGGER_CONFIG; +use crate::mmtk; +use crate::upcalls; +use crate::Ruby; +use mmtk::memory_manager; +use mmtk::scheduler::*; +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::VMMutatorThread; +use mmtk::util::VMThread; +use mmtk::util::VMWorkerThread; +use mmtk::vm::Collection; +use mmtk::vm::GCThreadContext; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::thread; + +static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false); + +pub struct VMCollection {} + +impl Collection<Ruby> for VMCollection { + fn is_collection_enabled() -> bool { + crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed) + } + + fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + (upcalls().stop_the_world)(); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed); + (upcalls().before_updating_jit_code)(); + } else { + CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed); + } + + crate::binding().pinning_registry.pin_children(tls); + + (upcalls().get_mutators)( + Self::notify_mutator_ready::<F>, + &mut mutator_visitor as *mut F as *mut _, + ); + } + + fn resume_mutators(_tls: VMWorkerThread) { + let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed); + + if current_gc_may_move { + (upcalls().after_updating_jit_code)(); + } + + (upcalls().resume_mutators)(current_gc_may_move); + } + + fn block_for_gc(tls: VMMutatorThread) { + (upcalls().block_for_gc)(tls); + } + + fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext<Ruby>) { + let join_handle = match ctx { + GCThreadContext::Worker(mut worker) => thread::Builder::new() + .name("MMTk Worker Thread".to_string()) + .spawn(move || { + let ordinal = worker.ordinal; + debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}"); + crate::register_gc_thread(thread::current().id()); + let ptr_worker = &mut *worker as *mut GCWorker<Ruby>; + let gc_thread_tls = + Box::into_raw(Box::new(GCThreadTLS::for_worker(ptr_worker))); + (upcalls().init_gc_worker_thread)(gc_thread_tls); + memory_manager::start_worker( + mmtk(), + GCThreadTLS::to_vwt(gc_thread_tls), + worker, + ); + debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}"); + crate::unregister_gc_thread(thread::current().id()); + }) + .unwrap(), + }; + + { + let mut handles = crate::binding().gc_thread_join_handles.lock().unwrap(); + handles.push(join_handle); + } + } + + fn vm_live_bytes() -> usize { + (upcalls().vm_live_bytes)() + } + + fn create_gc_trigger() -> Box<dyn GCTriggerPolicy<Ruby>> { + // `GCTriggerSelector::Delegated` is currently used by two different + // heap modes: `ruby` (the Ruby-like free-slot ratio trigger) and `cpu` + // (the CPU-overhead trigger from Tavakolisomeh et al., MPLR '23). + // Which one is active is determined by which `OnceCell` config the + // `MMTK_HEAP_MODE` parser populated. + if CPU_HEAP_TRIGGER_CONFIG.get().is_some() { + Box::new(CpuHeapTrigger::default()) + } else { + Box::new(RubyHeapTrigger::default()) + } + } +} + +impl VMCollection { + extern "C" fn notify_mutator_ready<F>(mutator_ptr: *mut RubyMutator, data: *mut libc::c_void) + where + F: FnMut(&'static mut mmtk::Mutator<Ruby>), + { + let mutator = unsafe { &mut *mutator_ptr }; + let mutator_visitor = unsafe { &mut *(data as *mut F) }; + mutator_visitor(mutator); + } +} diff --git a/gc/mmtk/src/heap/cpu_heap_trigger.rs b/gc/mmtk/src/heap/cpu_heap_trigger.rs new file mode 100644 index 0000000000..ef5a79fe9a --- /dev/null +++ b/gc/mmtk/src/heap/cpu_heap_trigger.rs @@ -0,0 +1,370 @@ +//! A GC trigger that adjusts the heap size based on the CPU overhead of GC. +//! +//! This is an implementation of the heap sizing policy described in +//! Tavakolisomeh, Shimchenko, Österlund, Bruno, Ferreira, Wrigstad, +//! "Heap Size Adjustment with CPU Control", MPLR '23. +//! <https://doi.org/10.1145/3617651.3622988> +//! +//! The idea: rather than letting heap size control GC frequency, let a +//! user-supplied *target GC CPU overhead* control the heap size. After each GC +//! cycle, we measure the GC CPU overhead (fraction of process CPU time spent +//! in GC) and compare it to the target. If GC is over budget we grow the heap +//! (reducing GC frequency); if it is under budget we shrink the heap (trading +//! memory for more frequent collections). +//! +//! ## Algorithm +//! +//! After each GC cycle we compute, using an average of the last `n` cycles: +//! +//! ```text +//! GC_CPU = T_GC / T_APP (Eq. 1) +//! overhead_error = GC_CPU - target (Eq. 2) +//! sigmoid_error = 1 / (1 + e^(-overhead_error)) (Eq. 3) +//! adjustment_factor = sigmoid_error + 0.5 (in (0.5, 1.5)) (Eq. 4) +//! new_size = current_size * adjustment_factor (Eq. 5) +//! ``` +//! +//! where: +//! - `T_GC` is the wall-clock duration of each GC cycle. +//! - `T_APP` is process CPU time elapsed between consecutive GC cycles (sum of +//! CPU time over all threads — mutators, GC workers, compilers, etc.), read +//! via `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`. +//! +//! The final heap size is then clamped to the range +//! `[max(1.1 * used, min_heap_pages), max_heap_pages]`, providing 10% headroom +//! above current live memory to avoid triggering GC on an effectively-empty +//! heap. +//! +//! ## Differences from the paper +//! +//! The paper targets ZGC, a concurrent generational collector. MMTk's Ruby +//! binding currently ships stop-the-world collectors (Immix, MarkSweep). The +//! paper's formula still applies: with a STW collector the process CPU time +//! during GC closely tracks the wall-clock GC time, and mutator CPU time +//! during the mutator phase is correctly attributed. For generational plans +//! we skip nursery-only GCs, consistent with MemBalancer. + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static CPU_HEAP_TRIGGER_CONFIG: OnceCell<CpuHeapTriggerConfig> = OnceCell::new(); + +/// Configuration for the [`CpuHeapTrigger`]. +pub struct CpuHeapTriggerConfig { + /// Lower bound on heap size (in pages). The trigger will never shrink below + /// this value. + pub min_heap_pages: usize, + /// Upper bound on heap size (in pages). The trigger will never grow above + /// this value. + pub max_heap_pages: usize, + /// Initial heap size (in pages). + pub initial_heap_pages: usize, + /// Target GC CPU overhead as a fraction of total process CPU time. For + /// example, `0.15` means the policy will try to keep GC CPU usage near 15%. + /// Valid range: `(0.0, 1.0)`. + pub target_gc_cpu: f64, + /// Number of recent GC cycles averaged together when computing the CPU + /// overhead signal. Smoothes out short-term fluctuations. The paper uses 3. + pub window_size: usize, +} + +/// A single GC cycle's timing measurements. +#[derive(Clone, Copy, Debug, Default)] +struct GcSample { + /// Wall-clock seconds spent inside this GC cycle. + gc_seconds: f64, + /// Seconds of process CPU time elapsed since the previous GC cycle ended. + /// This covers both mutator time and (on multi-threaded mutators) any + /// mutator CPU time consumed in parallel with the previous GC. + app_cpu_seconds: f64, +} + +struct CpuHeapTriggerState { + /// Ring buffer of the last `window_size` samples. Oldest-first. + samples: Vec<GcSample>, + /// Wall-clock time when the current GC cycle started. `None` when no GC is + /// in progress. + gc_start_wall: Option<std::time::Instant>, + /// Process CPU time (seconds) recorded at the end of the previous GC + /// cycle. `None` until the first cycle completes. + last_gc_end_cpu: Option<f64>, +} + +impl CpuHeapTriggerState { + fn new() -> Self { + Self { + samples: Vec::new(), + gc_start_wall: None, + last_gc_end_cpu: None, + } + } + + /// Pushes a new sample, dropping the oldest when the window is full. + fn push_sample(&mut self, sample: GcSample, window_size: usize) { + if self.samples.len() >= window_size { + self.samples.remove(0); + } + self.samples.push(sample); + } + + /// Returns the arithmetic mean GC CPU overhead across the window, or + /// `None` if we don't yet have a full sample (which happens on the first + /// GC cycle — we have no baseline for `app_cpu_seconds`). + fn mean_gc_cpu(&self) -> Option<f64> { + if self.samples.is_empty() { + return None; + } + let total_gc: f64 = self.samples.iter().map(|s| s.gc_seconds).sum(); + let total_app: f64 = self.samples.iter().map(|s| s.app_cpu_seconds).sum(); + if total_app <= 0.0 { + return None; + } + Some(total_gc / total_app) + } +} + +pub struct CpuHeapTrigger { + /// Target heap size in pages. Updated at the end of each GC cycle. + target_heap_pages: AtomicUsize, + /// Mutable timing state. Wrapped in a `Mutex` because `on_gc_start` and + /// `on_gc_end` are the only mutation sites and they are not on an + /// allocation hot path; avoiding the complexity of lock-free state is + /// worth the trivial contention. + state: Mutex<CpuHeapTriggerState>, +} + +impl Default for CpuHeapTrigger { + fn default() -> Self { + let cfg = Self::get_config(); + Self { + target_heap_pages: AtomicUsize::new(cfg.initial_heap_pages), + state: Mutex::new(CpuHeapTriggerState::new()), + } + } +} + +impl GCTriggerPolicy<Ruby> for CpuHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide, matching the other triggers. + plan.collection_required(space_full, space) + } + + fn on_gc_start(&self, _mmtk: &'static MMTK<Ruby>) { + let mut state = self.state.lock().unwrap(); + state.gc_start_wall = Some(std::time::Instant::now()); + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + // Skip nursery-only GCs for generational plans. The heap resizing + // decision is driven by the (much more expensive) full collections + // where the signal-to-noise ratio is high enough to be useful. + if let Some(gen_plan) = mmtk.get_plan().generational() { + if gen_plan.is_current_gc_nursery() { + return; + } + } + + let cfg = Self::get_config(); + let gc_end_cpu = process_cpu_time_seconds(); + + let mut state = self.state.lock().unwrap(); + + // Duration of this GC cycle (wall clock). + let gc_seconds = state + .gc_start_wall + .take() + .map(|start| start.elapsed().as_secs_f64()) + .unwrap_or(0.0); + + // Process CPU time elapsed since the previous GC cycle ended. We + // require at least one previous end timestamp to produce a valid + // sample — without it we cannot compute `T_APP`. + if let (Some(last_end), Some(now)) = (state.last_gc_end_cpu, gc_end_cpu) { + let app_cpu_seconds = (now - last_end).max(0.0); + // Only record non-degenerate samples to avoid poisoning the window + // with zero-time entries from back-to-back GCs. + if app_cpu_seconds > 0.0 { + state.push_sample( + GcSample { + gc_seconds, + app_cpu_seconds, + }, + cfg.window_size, + ); + } + } + state.last_gc_end_cpu = gc_end_cpu; + + // Compute the new heap size only when we have samples to average over. + if let Some(gc_cpu) = state.mean_gc_cpu() { + // Drop the lock before doing the (relatively cheap) math and + // atomic update; nothing below needs the state. + drop(state); + + let overhead_error = gc_cpu - cfg.target_gc_cpu; // Eq. (2) + let sigmoid_error = sigmoid(overhead_error); // Eq. (3) + let adjustment_factor = sigmoid_error + 0.5; // Eq. (4), range (0.5, 1.5) + + let current = self.target_heap_pages.load(Ordering::Relaxed); + let suggested = ((current as f64) * adjustment_factor) as usize; // Eq. (5) + + // Clamp: + // - upper bound: configured max + // - lower bound: max(1.1 * used, min) — 10% headroom above current + // live memory, so we never request a heap so small that GC is + // triggered immediately on return from this one. + let used = mmtk.get_plan().get_used_pages(); + let floor = ((used as f64) * 1.1).ceil() as usize; + let lower = floor.max(cfg.min_heap_pages).min(cfg.max_heap_pages); + let upper = cfg.max_heap_pages; + let new_target = suggested.clamp(lower, upper); + + self.target_heap_pages.store(new_target, Ordering::Relaxed); + + info!( + "CpuHeapTrigger: gc_cpu={:.4} target={:.4} factor={:.4} \ + pages {} -> {} (used={}, clamp=[{}, {}])", + gc_cpu, + cfg.target_gc_cpu, + adjustment_factor, + current, + new_target, + used, + lower, + upper + ); + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl CpuHeapTrigger { + fn get_config<'b>() -> &'b CpuHeapTriggerConfig { + CPU_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use CPU_HEAP_TRIGGER_CONFIG before it is initialized") + } +} + +/// Standard logistic sigmoid. Returns 0.5 when x == 0, asymptotes to 0 and 1. +fn sigmoid(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +/// Reads the process-wide CPU time as a floating-point number of seconds, +/// summed across all threads of this process. Returns `None` if the clock +/// query fails (which should be essentially impossible on supported +/// platforms). +fn process_cpu_time_seconds() -> Option<f64> { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: `clock_gettime` writes exactly `sizeof(timespec)` bytes to the + // pointer we pass, which is a valid local stack allocation. + let rc = unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) }; + if rc != 0 { + return None; + } + Some((ts.tv_sec as f64) + (ts.tv_nsec as f64) / 1_000_000_000.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sigmoid_is_well_behaved() { + assert!((sigmoid(0.0) - 0.5).abs() < 1e-12); + assert!(sigmoid(-100.0) < 1e-9); + assert!(sigmoid(100.0) > 1.0 - 1e-9); + // Monotonic. + assert!(sigmoid(-1.0) < sigmoid(0.0)); + assert!(sigmoid(0.0) < sigmoid(1.0)); + } + + #[test] + fn adjustment_factor_is_within_paper_bounds() { + // Eq. (4): adjustment_factor = sigmoid(e) + 0.5 must lie in (0.5, 1.5). + for e in [-10.0_f64, -1.0, 0.0, 1.0, 10.0] { + let f = sigmoid(e) + 0.5; + assert!(f > 0.5 && f < 1.5, "factor {f} out of range for e={e}"); + } + } + + #[test] + fn mean_gc_cpu_is_total_weighted() { + let mut state = CpuHeapTriggerState::new(); + state.push_sample( + GcSample { + gc_seconds: 1.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + state.push_sample( + GcSample { + gc_seconds: 3.0, + app_cpu_seconds: 10.0, + }, + 3, + ); + // (1 + 3) / (10 + 10) = 0.2 + assert!((state.mean_gc_cpu().unwrap() - 0.2).abs() < 1e-12); + } + + #[test] + fn window_drops_oldest() { + let mut state = CpuHeapTriggerState::new(); + for i in 0..5 { + state.push_sample( + GcSample { + gc_seconds: i as f64, + app_cpu_seconds: 1.0, + }, + 3, + ); + } + assert_eq!(state.samples.len(), 3); + // After pushing 0,1,2,3,4 with window 3, we should have [2,3,4]. + assert_eq!(state.samples[0].gc_seconds, 2.0); + assert_eq!(state.samples[2].gc_seconds, 4.0); + } + + #[test] + fn no_sample_without_prior_gc() { + // First GC cycle cannot produce a sample (no previous end time). The + // push happens only when last_gc_end_cpu is Some. + let state = CpuHeapTriggerState::new(); + assert!(state.mean_gc_cpu().is_none()); + } +} diff --git a/gc/mmtk/src/heap/mod.rs b/gc/mmtk/src/heap/mod.rs new file mode 100644 index 0000000000..05a35efb23 --- /dev/null +++ b/gc/mmtk/src/heap/mod.rs @@ -0,0 +1,9 @@ +mod cpu_heap_trigger; +mod ruby_heap_trigger; + +pub use cpu_heap_trigger::CpuHeapTrigger; +pub use cpu_heap_trigger::CpuHeapTriggerConfig; +pub use cpu_heap_trigger::CPU_HEAP_TRIGGER_CONFIG; +pub use ruby_heap_trigger::RubyHeapTrigger; +pub use ruby_heap_trigger::RubyHeapTriggerConfig; +pub use ruby_heap_trigger::RUBY_HEAP_TRIGGER_CONFIG; diff --git a/gc/mmtk/src/heap/ruby_heap_trigger.rs b/gc/mmtk/src/heap/ruby_heap_trigger.rs new file mode 100644 index 0000000000..fe1130043d --- /dev/null +++ b/gc/mmtk/src/heap/ruby_heap_trigger.rs @@ -0,0 +1,105 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use mmtk::util::heap::GCTriggerPolicy; +use mmtk::util::heap::SpaceStats; +use mmtk::Plan; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +use crate::Ruby; + +pub static RUBY_HEAP_TRIGGER_CONFIG: OnceCell<RubyHeapTriggerConfig> = OnceCell::new(); + +pub struct RubyHeapTriggerConfig { + /// Min heap size + pub min_heap_pages: usize, + /// Max heap size + pub max_heap_pages: usize, + /// Minimum ratio of empty space after a GC before the heap will grow + pub heap_pages_min_ratio: f64, + /// Ratio the heap will grow by + pub heap_pages_goal_ratio: f64, + /// Maximum ratio of empty space after a GC before the heap will shrink + pub heap_pages_max_ratio: f64, +} + +pub struct RubyHeapTrigger { + /// Target number of heap pages + target_heap_pages: AtomicUsize, +} + +impl GCTriggerPolicy<Ruby> for RubyHeapTrigger { + fn is_gc_required( + &self, + space_full: bool, + space: Option<SpaceStats<Ruby>>, + plan: &dyn Plan<VM = Ruby>, + ) -> bool { + // Let the plan decide + plan.collection_required(space_full, space) + } + + fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) { + if let Some(plan) = mmtk.get_plan().generational() { + if plan.is_current_gc_nursery() { + // Nursery GC + } else { + // Full GC + } + + panic!("TODO: support for generational GC not implemented") + } else { + let used_pages = mmtk.get_plan().get_used_pages(); + + let target_min = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_min_ratio)) as usize; + let target_max = + (used_pages as f64 * (1.0 + Self::get_config().heap_pages_max_ratio)) as usize; + let new_target = + (((used_pages as f64) * (1.0 + Self::get_config().heap_pages_goal_ratio)) as usize) + .clamp( + Self::get_config().min_heap_pages, + Self::get_config().max_heap_pages, + ); + + if used_pages < target_min || used_pages > target_max { + self.target_heap_pages.store(new_target, Ordering::Relaxed); + } + } + } + + fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool { + plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_current_heap_size_in_pages(&self) -> usize { + self.target_heap_pages.load(Ordering::Relaxed) + } + + fn get_max_heap_size_in_pages(&self) -> usize { + Self::get_config().max_heap_pages + } + + fn can_heap_size_grow(&self) -> bool { + self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages + } +} + +impl Default for RubyHeapTrigger { + fn default() -> Self { + let min_heap_pages = Self::get_config().min_heap_pages; + + Self { + target_heap_pages: AtomicUsize::new(min_heap_pages), + } + } +} + +impl RubyHeapTrigger { + fn get_config<'b>() -> &'b RubyHeapTriggerConfig { + RUBY_HEAP_TRIGGER_CONFIG + .get() + .expect("Attempt to use RUBY_HEAP_TRIGGER_CONFIG before it is initialized") + } +} diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs new file mode 100644 index 0000000000..52dc782051 --- /dev/null +++ b/gc/mmtk/src/lib.rs @@ -0,0 +1,161 @@ +// Warn about unsafe operations in functions that are already marked as unsafe. +// This will become default in Rust 2024 edition. +#![warn(unsafe_op_in_unsafe_fn)] + +extern crate libc; +extern crate mmtk; +#[macro_use] +extern crate log; +extern crate probe; + +use std::collections::HashSet; +use std::panic::PanicHookInfo; +use std::sync::Mutex; +use std::thread::ThreadId; + +use abi::RubyUpcalls; +use binding::RubyBinding; +use binding::RubyBindingFast; +use binding::RubyConfiguration; +use mmtk::vm::slot::SimpleSlot; +use mmtk::vm::slot::UnimplementedMemorySlice; +use mmtk::vm::VMBinding; +use mmtk::MMTK; +use once_cell::sync::OnceCell; + +pub mod abi; +pub mod active_plan; +pub mod api; +pub mod binding; +pub mod collection; +pub mod heap; +pub mod object_model; +pub mod pinning_registry; +pub mod reference_glue; +pub mod scanning; +pub mod utils; +pub mod weak_proc; + +#[derive(Default)] +pub struct Ruby; + +/// Ruby slot type, i.e. a slot that holds a VALUE. +/// Currently we use SimpleSlot. +/// It doesn't matter, becaues we have not started using slot-enqueuing, yet. +pub type RubySlot = SimpleSlot; + +/// Ruby memory slice, i.e. an array of VALUEs. +/// It is used by array-copy barriers which is supposed to perform bettern than copying array +/// elements one by one. At this moment, we just leave it unimplemented. +pub type RubyMemorySlice = UnimplementedMemorySlice<RubySlot>; + +impl VMBinding for Ruby { + type VMObjectModel = object_model::VMObjectModel; + type VMScanning = scanning::VMScanning; + type VMCollection = collection::VMCollection; + type VMActivePlan = active_plan::VMActivePlan; + type VMReferenceGlue = reference_glue::VMReferenceGlue; + + type VMSlot = RubySlot; + type VMMemorySlice = RubyMemorySlice; +} + +/// The callback for mutator thread panic handler (which calls rb_bug to output +/// debugging information such as the Ruby backtrace and memory maps). +/// This is set before BINDING is set because mmtk_init could panic. +pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell<extern "C" fn()> = OnceCell::new(); + +/// The singleton object for the Ruby binding itself. +pub static BINDING: OnceCell<RubyBinding> = OnceCell::new(); + +/// Some data needs to be accessed fast. +/// We sacrifice safety for speed using unsynchronized global variables. +pub static mut BINDING_FAST: RubyBindingFast = RubyBindingFast::new(); + +/// Some data needs to be accessed fast. +pub static CONFIGURATION: RubyConfiguration = RubyConfiguration::new(); + +pub fn binding<'b>() -> &'b RubyBinding { + BINDING + .get() + .expect("Attempt to use the binding before it is initialization") +} + +pub fn mmtk() -> &'static MMTK<Ruby> { + binding().mmtk +} + +pub fn upcalls() -> &'static RubyUpcalls { + binding().upcalls() +} + +pub static GC_THREADS: OnceCell<Mutex<HashSet<ThreadId>>> = OnceCell::new(); + +pub(crate) fn register_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.insert(thread_id); +} + +pub(crate) fn unregister_gc_thread(thread_id: ThreadId) { + let mut gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.remove(&thread_id); +} + +pub(crate) fn is_gc_thread(thread_id: ThreadId) -> bool { + let gc_threads = GC_THREADS.get().unwrap().lock().unwrap(); + gc_threads.contains(&thread_id) +} + +fn handle_gc_thread_panic(panic_info: &PanicHookInfo) { + eprintln!("ERROR: An MMTk GC thread panicked. This is a bug."); + eprintln!("{panic_info}"); + + let bt = std::backtrace::Backtrace::capture(); + match bt.status() { + std::backtrace::BacktraceStatus::Unsupported => { + eprintln!("Backtrace is unsupported.") + } + std::backtrace::BacktraceStatus::Disabled => { + eprintln!("Backtrace is disabled."); + eprintln!("run with `RUST_BACKTRACE=1` environment variable to display a backtrace"); + } + std::backtrace::BacktraceStatus::Captured => { + eprintln!("{bt}"); + } + s => { + eprintln!("Unknown backtrace status: {s:?}"); + } + } +} + +pub(crate) fn set_panic_hook() { + if GC_THREADS.set(Default::default()).is_err() { + return; + } + + let old_hook = std::panic::take_hook(); + + std::panic::set_hook(Box::new(move |panic_info| { + if is_gc_thread(std::thread::current().id()) { + handle_gc_thread_panic(panic_info); + + (crate::binding().upcalls().gc_thread_panic_handler)(); + } else { + old_hook(panic_info); + (crate::MUTATOR_THREAD_PANIC_HANDLER + .get() + .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))(); + } + })); +} + +/// This kind of assertion is enabled if either building in debug mode or the +/// "extra_assert" feature is enabled. +#[macro_export] +macro_rules! extra_assert { + ($($arg:tt)*) => { + if std::cfg!(any(debug_assertions, feature = "extra_assert")) { + std::assert!($($arg)*); + } + }; +} diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs new file mode 100644 index 0000000000..d673ca11a0 --- /dev/null +++ b/gc/mmtk/src/object_model.rs @@ -0,0 +1,124 @@ +use std::ptr::copy_nonoverlapping; + +use crate::abi; +use crate::abi::RubyObjectAccess; +use crate::abi::MIN_OBJ_ALIGN; +use crate::abi::OBJREF_OFFSET; +use crate::Ruby; +use mmtk::util::constants::BITS_IN_BYTE; +use mmtk::util::copy::CopySemantics; +use mmtk::util::copy::GCWorkerCopyContext; +use mmtk::util::Address; +use mmtk::util::ObjectReference; +use mmtk::vm::*; + +pub struct VMObjectModel {} + +impl VMObjectModel { + const OBJREF_OFFSET: usize = abi::OBJREF_OFFSET; +} + +impl ObjectModel<Ruby> for VMObjectModel { + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); + + // We overwrite the prepended word which were used to hold object sizes. + const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = + VMLocalForwardingPointerSpec::in_header(-((OBJREF_OFFSET * BITS_IN_BYTE) as isize)); + + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_first(); + + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = + VMLocalMarkBitSpec::side_after(Self::LOCAL_FORWARDING_BITS_SPEC.as_spec()); + + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = + VMLocalPinningBitSpec::side_after(Self::LOCAL_MARK_BIT_SPEC.as_spec()); + + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = + VMLocalLOSMarkNurserySpec::side_after(Self::LOCAL_PINNING_BIT_SPEC.as_spec()); + + const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; + const OBJECT_REF_OFFSET_LOWER_BOUND: isize = Self::OBJREF_OFFSET as isize; + + const NEED_VO_BITS_DURING_TRACING: bool = true; + + fn copy( + from: ObjectReference, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext<Ruby>, + ) -> ObjectReference { + let from_acc = RubyObjectAccess::from_objref(from); + let from_start = from_acc.obj_start(); + let object_size = from_acc.object_size(); + let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics); + debug_assert!(!to_start.is_zero()); + let to_payload = to_start.add(OBJREF_OFFSET); + unsafe { + copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size); + } + let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) }; + copy_context.post_copy(to_obj, object_size, semantics); + trace!("Copied object from {} to {}", from, to_obj); + + (crate::binding().upcalls().move_obj_during_marking)(from, to_obj); + + #[cfg(feature = "clear_old_copy")] + { + trace!( + "Clearing old copy {} ({}-{})", + from, + from_start, + from_start + object_size + ); + // For debug purpose, we clear the old copy so that if the Ruby VM reads from the old + // copy again, it will likely result in an error. + unsafe { std::ptr::write_bytes::<u8>(from_start.to_mut_ptr(), 0, object_size) } + } + + to_obj + } + + fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_reference_when_copied_to(_from: ObjectReference, _to: Address) -> ObjectReference { + unimplemented!( + "This function cannot be called because we do not support MarkCompact for Ruby." + ) + } + + fn get_current_size(object: ObjectReference) -> usize { + RubyObjectAccess::from_objref(object).object_size() + } + + fn get_type_descriptor(_reference: ObjectReference) -> &'static [i8] { + todo!() + } + + fn ref_to_object_start(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).obj_start() + } + + fn ref_to_header(object: ObjectReference) -> Address { + RubyObjectAccess::from_objref(object).payload_addr() + } + + fn get_size_when_copied(object: ObjectReference) -> usize { + Self::get_current_size(object) + } + + fn get_align_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn get_align_offset_when_copied(_object: ObjectReference) -> usize { + todo!() + } + + fn dump_object(_object: ObjectReference) { + todo!() + } +} diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs new file mode 100644 index 0000000000..b498b508f1 --- /dev/null +++ b/gc/mmtk/src/pinning_registry.rs @@ -0,0 +1,187 @@ +use std::sync::Mutex; + +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::MMTK; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct PinningRegistry { + pinning_objs: Mutex<Vec<ObjectReference>>, + pinned_objs: Mutex<Vec<ObjectReference>>, +} + +impl PinningRegistry { + pub fn new() -> Self { + Self { + pinning_objs: Default::default(), + pinned_objs: Default::default(), + } + } + + pub fn register(&self, object: ObjectReference) { + let mut pinning_objs = self.pinning_objs.lock().unwrap(); + pinning_objs.push(object); + } + + pub fn pin_children(&self, tls: VMWorkerThread) { + if !crate::mmtk().get_plan().current_gc_may_move_object() { + log::debug!("The current GC is non-moving, skipping pinning children."); + return; + } + + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let worker = gc_tls.worker(); + + let pinning_objs = self + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + let packet_size = 512; + let work_packets = pinning_objs + .chunks(packet_size) + .map(|chunk| { + Box::new(PinPinningChildren { + pinning_objs: chunk.to_vec(), + }) as _ + }) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(work_packets); + } + + pub fn cleanup(&self, worker: &mut GCWorker<Ruby>) { + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(RemoveDeadPinnings); + if crate::mmtk().get_plan().current_gc_may_move_object() { + let packet = { + let mut pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + UnpinPinnedObjects { + objs: std::mem::take(&mut pinned_objs), + } + }; + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(packet); + } else { + debug!("The current GC is non-moving, skipping unpinning objects."); + debug_assert_eq!( + { + let pinned_objs = self + .pinned_objs + .try_lock() + .expect("Unexpected contention on pinned_objs"); + pinned_objs.len() + }, + 0 + ); + } + } +} + +impl Default for PinningRegistry { + fn default() -> Self { + Self::new() + } +} + +struct PinPinningChildren { + pinning_objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for PinPinningChildren { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + let mut pinned_objs = vec![]; + let mut newly_pinned_objs = vec![]; + + let visit_object = |_worker, target_object: ObjectReference, pin| { + log::trace!( + " -> {} {}", + if pin { "(pin)" } else { " " }, + target_object + ); + if pin { + debug_assert!( + target_object.get_forwarded_object().is_none(), + "Trying to pin {target_object} but has been moved" + ); + + pinned_objs.push(target_object); + } + target_object + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + for obj in self.pinning_objs.iter().cloned() { + log::trace!(" Pinning: {}", obj); + (upcalls().call_gc_mark_children)(obj); + } + }); + + for target_object in pinned_objs { + if memory_manager::pin_object(target_object) { + newly_pinned_objs.push(target_object); + } + } + + let mut pinned_objs = crate::binding() + .pinning_registry + .pinned_objs + .lock() + .unwrap(); + pinned_objs.append(&mut newly_pinned_objs); + } +} + +struct RemoveDeadPinnings; + +impl GCWork<Ruby> for RemoveDeadPinnings { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Removing dead Pinnings..."); + + let registry = &crate::binding().pinning_registry; + { + let mut pinning_objs = registry + .pinning_objs + .try_lock() + .expect("PinningRegistry should not have races during GC."); + + pinning_objs.retain_mut(|obj| { + if obj.is_live() { + let new_obj = obj.get_forwarded_object().unwrap_or(*obj); + *obj = new_obj; + true + } else { + log::trace!(" Dead Pinning removed: {}", *obj); + false + } + }); + } + } +} + +struct UnpinPinnedObjects { + objs: Vec<ObjectReference>, +} + +impl GCWork<Ruby> for UnpinPinnedObjects { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) { + log::debug!("Unpinning pinned objects..."); + + for obj in self.objs.iter() { + let unpinned = memory_manager::unpin_object(*obj); + debug_assert!(unpinned); + } + } +} diff --git a/gc/mmtk/src/reference_glue.rs b/gc/mmtk/src/reference_glue.rs new file mode 100644 index 0000000000..1272bd54c1 --- /dev/null +++ b/gc/mmtk/src/reference_glue.rs @@ -0,0 +1,26 @@ +use crate::Ruby; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ReferenceGlue; + +pub struct VMReferenceGlue {} + +impl ReferenceGlue<Ruby> for VMReferenceGlue { + type FinalizableType = ObjectReference; + + fn get_referent(_object: ObjectReference) -> Option<ObjectReference> { + unimplemented!() + } + + fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { + unimplemented!() + } + + fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) { + unimplemented!() + } + + fn clear_referent(_new_reference: ObjectReference) { + unimplemented!() + } +} diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs new file mode 100644 index 0000000000..355a2e7759 --- /dev/null +++ b/gc/mmtk/src/scanning.rs @@ -0,0 +1,291 @@ +use crate::abi::GCThreadTLS; + +use crate::upcalls; +use crate::utils::ChunkedVecCollector; +use crate::Ruby; +use crate::RubySlot; +use mmtk::memory_manager; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::util::VMWorkerThread; +use mmtk::vm::ObjectTracer; +use mmtk::vm::RootsWorkFactory; +use mmtk::vm::Scanning; +use mmtk::vm::SlotVisitor; +use mmtk::Mutator; + +pub struct VMScanning {} + +impl Scanning<Ruby> for VMScanning { + const UNIQUE_OBJECT_ENQUEUING: bool = true; + + fn support_slot_enqueuing(_tls: VMWorkerThread, _object: ObjectReference) -> bool { + false + } + + fn scan_object<EV: SlotVisitor<RubySlot>>( + _tls: VMWorkerThread, + _object: ObjectReference, + _slot_visitor: &mut EV, + ) { + unreachable!("We have not enabled slot enqueuing for any types, yet."); + } + + fn scan_object_and_trace_edges<OT: ObjectTracer>( + tls: VMWorkerThread, + object: ObjectReference, + object_tracer: &mut OT, + ) { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Not an MMTk object: {object}", + ); + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let visit_object = |_worker, target_object: ObjectReference, pin| { + trace!( + "Tracing edge: {} -> {}{}", + object, + target_object, + if pin { " pin" } else { "" } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object. Src: {object} dst: {target_object}" + ); + + debug_assert!( + // If we are in a moving GC, all objects should be pinned by PinningRegistry. + // If it is requested that target_object be pinned but it is not pinned, then + // it is a bug because it could be moved. + if crate::mmtk().get_plan().current_gc_may_move_object() && pin { + memory_manager::is_pinned(target_object) + } else { + true + }, + "Object {object} is trying to pin {target_object}" + ); + + let forwarded_target = object_tracer.trace_object(target_object); + if forwarded_target != target_object { + trace!(" Forwarded target {target_object} -> {forwarded_target}"); + } + forwarded_target + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + }); + } + + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { + // Do nothing + } + + fn scan_roots_in_mutator_thread( + _tls: VMWorkerThread, + _mutator: &'static mut Mutator<Ruby>, + mut _factory: impl RootsWorkFactory<RubySlot>, + ) { + // Do nothing. All stacks (including Ruby stacks and machine stacks) are reachable from + // `rb_vm_t` -> ractor -> thread -> fiber -> stacks. It is part of `ScanGCRoots` which + // calls `rb_gc_mark_roots` -> `rb_vm_mark`. + } + + fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory<RubySlot>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) }; + let root_scanning_work_packets: Vec<Box<dyn GCWork<Ruby>>> = vec![ + Box::new(ScanGCRoots::new(factory.clone())), + Box::new(ScanObjspace::new(factory.clone())), + ]; + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare] + .bulk_add(root_scanning_work_packets); + + // Generate WB-unprotected roots scanning work packets + + 'gen_wb_unprotected_work: { + let is_nursery_gc = (crate::mmtk().get_plan().generational()) + .is_some_and(|gen| gen.is_current_gc_nursery()); + if !is_nursery_gc { + break 'gen_wb_unprotected_work; + } + + let vecs = { + let guard = crate::binding() + .wb_unprotected_objects + .try_lock() + .expect("Someone is holding the lock of wb_unprotected_objects?"); + if guard.is_empty() { + break 'gen_wb_unprotected_work; + } + + let mut collector = ChunkedVecCollector::new(128); + collector.extend(guard.iter().copied()); + collector.into_vecs() + }; + + let packets = vecs + .into_iter() + .map(|objects| { + let factory = factory.clone(); + Box::new(ScanWbUnprotectedRoots { factory, objects }) as _ + }) + .collect::<Vec<_>>(); + + gc_tls.worker().scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(packets); + } + } + + fn supports_return_barrier() -> bool { + false + } + + fn prepare_for_roots_re_scanning() { + todo!() + } + + fn process_weak_refs( + worker: &mut GCWorker<Ruby>, + tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) -> bool { + crate::binding() + .weak_proc + .process_weak_stuff(worker, tracer_context); + crate::binding().pinning_registry.cleanup(worker); + false + } + + fn forward_weak_refs( + _worker: &mut GCWorker<Ruby>, + _tracer_context: impl mmtk::vm::ObjectTracerContext<Ruby>, + ) { + panic!("We can't use MarkCompact in Ruby."); + } +} + +impl VMScanning { + const OBJECT_BUFFER_SIZE: usize = 4096; + + fn collect_object_roots_in<F: FnOnce()>( + root_scan_kind: &str, + gc_tls: &mut GCThreadTLS, + factory: &mut impl RootsWorkFactory<RubySlot>, + callback: F, + ) { + let mut buffer: Vec<ObjectReference> = Vec::new(); + let visit_object = |_, object: ObjectReference, pin| { + debug!( + "[{}] Visiting object: {}{}", + root_scan_kind, + object, + if pin { + "(unmovable root)" + } else { + "(movable, but we pin it anyway)" + } + ); + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "Root does not point to MMTk object. object: {object}" + ); + buffer.push(object); + if buffer.len() >= Self::OBJECT_BUFFER_SIZE { + factory.create_process_pinning_roots_work(std::mem::take(&mut buffer)); + } + object + }; + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, callback); + + if !buffer.is_empty() { + factory.create_process_pinning_roots_work(buffer); + } + } +} + +trait GlobaRootScanningWork { + type F: RootsWorkFactory<RubySlot>; + const NAME: &'static str; + + fn new(factory: Self::F) -> Self; + fn scan_roots(); + fn roots_work_factory(&mut self) -> &mut Self::F; + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let factory = self.roots_work_factory(); + + VMScanning::collect_object_roots_in(Self::NAME, gc_tls, factory, || { + Self::scan_roots(); + }); + } +} + +macro_rules! define_global_root_scanner { + ($name: ident, $code: expr) => { + struct $name<F: RootsWorkFactory<RubySlot>> { + factory: F, + } + impl<F: RootsWorkFactory<RubySlot>> GlobaRootScanningWork for $name<F> { + type F = F; + const NAME: &'static str = stringify!($name); + fn new(factory: Self::F) -> Self { + Self { factory } + } + fn scan_roots() { + $code + } + fn roots_work_factory(&mut self) -> &mut Self::F { + &mut self.factory + } + } + impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for $name<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobaRootScanningWork::do_work(self, worker, mmtk); + } + } + }; +} + +define_global_root_scanner!(ScanGCRoots, { + (crate::upcalls().scan_gc_roots)(); +}); + +define_global_root_scanner!(ScanObjspace, { + (crate::upcalls().scan_objspace)(); +}); + +struct ScanWbUnprotectedRoots<F: RootsWorkFactory<RubySlot>> { + factory: F, + objects: Vec<ObjectReference>, +} + +impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || { + for object in self.objects.iter().copied() { + if object.is_reachable() { + debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}"); + (upcalls().call_gc_mark_children)(object); + + if crate::mmtk().get_plan().current_gc_may_move_object() { + (upcalls().update_object_references)(object); + } + } else { + debug!( + "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}" + ); + } + } + }); + } +} diff --git a/gc/mmtk/src/utils.rs b/gc/mmtk/src/utils.rs new file mode 100644 index 0000000000..d1979eaf58 --- /dev/null +++ b/gc/mmtk/src/utils.rs @@ -0,0 +1,161 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use atomic_refcell::AtomicRefCell; +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; + +use crate::Ruby; +use sysinfo::System; + +pub struct ChunkedVecCollector<T> { + vecs: Vec<Vec<T>>, + current_vec: Vec<T>, + chunk_size: usize, +} + +impl<T> ChunkedVecCollector<T> { + pub fn new(chunk_size: usize) -> Self { + Self { + vecs: vec![], + current_vec: Vec::with_capacity(chunk_size), + chunk_size, + } + } + + pub fn add(&mut self, item: T) { + self.current_vec.push(item); + if self.current_vec.len() == self.chunk_size { + self.flush(); + } + } + + fn flush(&mut self) { + let new_vec = Vec::with_capacity(self.chunk_size); + let old_vec = std::mem::replace(&mut self.current_vec, new_vec); + self.vecs.push(old_vec); + } + + pub fn into_vecs(mut self) -> Vec<Vec<T>> { + if !self.current_vec.is_empty() { + self.flush(); + } + self.vecs + } +} + +impl<A> Extend<A> for ChunkedVecCollector<A> { + fn extend<T: IntoIterator<Item = A>>(&mut self, iter: T) { + for item in iter { + self.add(item); + } + } +} + +pub struct AfterAll { + counter: AtomicUsize, + stage: WorkBucketStage, + packets: AtomicRefCell<Vec<Box<dyn GCWork<Ruby>>>>, +} + +unsafe impl Sync for AfterAll {} + +impl AfterAll { + pub fn new(stage: WorkBucketStage) -> Self { + Self { + counter: AtomicUsize::new(0), + stage, + packets: AtomicRefCell::new(vec![]), + } + } + + pub fn add_packets(&self, mut packets: Vec<Box<dyn GCWork<Ruby>>>) { + let mut borrow = self.packets.borrow_mut(); + borrow.append(&mut packets); + } + + pub fn count_up(&self, n: usize) { + self.counter.fetch_add(n, Ordering::SeqCst); + } + + pub fn count_down(&self, worker: &mut GCWorker<Ruby>) { + let old = self.counter.fetch_sub(1, Ordering::SeqCst); + if old == 1 { + let packets = { + let mut borrow = self.packets.borrow_mut(); + std::mem::take(borrow.as_mut()) + }; + worker.scheduler().work_buckets[self.stage].bulk_add(packets); + } + } +} + +pub fn default_heap_max() -> usize { + let mut s = System::new(); + s.refresh_memory(); + s.total_memory() + .checked_mul(80) + .and_then(|v| v.checked_div(100)) + .expect("Invalid Memory size") as usize +} + +pub fn parse_capacity(input: &str) -> Option<usize> { + let trimmed = input.trim(); + + const KIBIBYTE: usize = 1024; + const MEBIBYTE: usize = 1024 * KIBIBYTE; + const GIBIBYTE: usize = 1024 * MEBIBYTE; + + let (number, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) { + trimmed.split_at(pos) + } else { + (trimmed, "") + }; + + let Ok(v) = number.parse::<usize>() else { + return None; + }; + + match suffix { + "GiB" => Some(v * GIBIBYTE), + "MiB" => Some(v * MEBIBYTE), + "KiB" => Some(v * KIBIBYTE), + "" => Some(v), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_capacity_parses_bare_bytes() { + assert_eq!(Some(1234), parse_capacity("1234")); + } + + #[test] + fn test_parse_capacity_parses_kibibytes() { + assert_eq!(Some(10240), parse_capacity("10KiB")); + } + + #[test] + fn test_parse_capacity_parses_mebibytes() { + assert_eq!(Some(10485760), parse_capacity("10MiB")) + } + + #[test] + fn test_parse_capacity_parses_gibibytes() { + assert_eq!(Some(10737418240), parse_capacity("10GiB")) + } + + #[test] + fn test_parse_capacity_parses_nonsense_values() { + assert_eq!(None, parse_capacity("notanumber")); + assert_eq!(None, parse_capacity("5tartswithanumber")); + assert_eq!(None, parse_capacity("number1nthemiddle")); + assert_eq!(None, parse_capacity("numberattheend111")); + assert_eq!(None, parse_capacity("mult1pl3numb3r5")); + } +} diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs new file mode 100644 index 0000000000..d38dbe04a4 --- /dev/null +++ b/gc/mmtk/src/weak_proc.rs @@ -0,0 +1,328 @@ +use std::sync::Mutex; + +use mmtk::scheduler::GCWork; +use mmtk::scheduler::GCWorker; +use mmtk::scheduler::WorkBucketStage; +use mmtk::util::ObjectReference; +use mmtk::vm::ObjectTracerContext; + +use crate::abi::GCThreadTLS; +use crate::upcalls; +use crate::Ruby; + +pub struct WeakProcessor { + non_parallel_obj_free_candidates: Mutex<Vec<ObjectReference>>, + parallel_obj_free_candidates: Vec<Mutex<Vec<ObjectReference>>>, + + /// Objects that needs `obj_free` called when dying. + /// If it is a bottleneck, replace it with a lock-free data structure, + /// or add candidates in batch. + weak_references: Mutex<Vec<ObjectReference>>, +} + +impl Default for WeakProcessor { + fn default() -> Self { + Self::new() + } +} + +impl WeakProcessor { + pub fn new() -> Self { + Self { + non_parallel_obj_free_candidates: Mutex::new(Vec::new()), + parallel_obj_free_candidates: vec![Mutex::new(Vec::new())], + weak_references: Mutex::new(Vec::new()), + } + } + + pub fn init_parallel_obj_free_candidates(&mut self, num_workers: usize) { + debug_assert_eq!(self.parallel_obj_free_candidates.len(), 1); + + for _ in 1..num_workers { + self.parallel_obj_free_candidates + .push(Mutex::new(Vec::new())); + } + } + + /// Add a batch of objects as candidates for `obj_free`. + /// + /// Amortizes mutex acquisition over the entire batch. Called when a + /// mutator's local buffer is flushed (buffer full or stop-the-world). + pub fn add_obj_free_candidates_batch( + &self, + objects: &[ObjectReference], + can_parallel_free: bool, + ) { + if objects.is_empty() { + return; + } + + if can_parallel_free { + let num_buckets = self.parallel_obj_free_candidates.len(); + for idx in 0..num_buckets { + let mut bucket = self.parallel_obj_free_candidates[idx].lock().unwrap(); + for (i, &obj) in objects.iter().enumerate() { + if i % num_buckets == idx { + bucket.push(obj); + } + } + } + } else { + self.non_parallel_obj_free_candidates + .lock() + .unwrap() + .extend_from_slice(objects); + } + } + + pub fn get_all_obj_free_candidates(&self) -> Vec<ObjectReference> { + // let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap(); + let mut all_obj_free_candidates = self + .non_parallel_obj_free_candidates + .lock() + .unwrap() + .to_vec(); + + for candidates_mutex in &self.parallel_obj_free_candidates { + all_obj_free_candidates.extend(candidates_mutex.lock().unwrap().to_vec()); + } + + std::mem::take(all_obj_free_candidates.as_mut()) + } + + pub fn add_weak_reference(&self, object: ObjectReference) { + let mut weak_references = self.weak_references.lock().unwrap(); + weak_references.push(object); + } + + pub fn weak_references_count(&self) -> usize { + self.weak_references.lock().unwrap().len() + } + + pub fn process_weak_stuff( + &self, + worker: &mut GCWorker<Ruby>, + _tracer_context: impl ObjectTracerContext<Ruby>, + ) { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessNonParallelObjFreeCanadidates {}, + ); + + for index in 0..self.parallel_obj_free_candidates.len() { + worker.add_work( + WorkBucketStage::VMRefClosure, + ProcessParallelObjFreeCandidates { index }, + ); + } + + worker.add_work(WorkBucketStage::VMRefClosure, ProcessWeakReferences); + + worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerObjIdTables); + + let global_tables_count = (crate::upcalls().global_tables_count)(); + let work_packets = (0..global_tables_count) + .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _) + .collect(); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(work_packets); + + worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure] + .bulk_add(vec![Box::new(UpdateWbUnprotectedObjectsList) as _]); + } +} + +fn process_obj_free_candidates(obj_free_candidates: &mut Vec<ObjectReference>) { + // Process obj_free + let mut new_candidates = Vec::new(); + + for object in obj_free_candidates.iter().copied() { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding obj_free candidate: {object} -> {new_object}"); + new_candidates.push(new_object); + } else { + (upcalls().call_obj_free)(object); + } + } + + *obj_free_candidates = new_candidates; +} + +struct ProcessParallelObjFreeCandidates { + index: usize, +} + +impl GCWork<Ruby> for ProcessParallelObjFreeCandidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding().weak_proc.parallel_obj_free_candidates + [self.index] + .try_lock() + .expect("Lock for parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessNonParallelObjFreeCanadidates; + +impl GCWork<Ruby> for ProcessNonParallelObjFreeCanadidates { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut obj_free_candidates = crate::binding() + .weak_proc + .non_parallel_obj_free_candidates + .try_lock() + .expect("Lock for non_parallel_obj_free_candidates should not be held"); + + process_obj_free_candidates(&mut obj_free_candidates); + } +} + +struct ProcessWeakReferences; + +impl GCWork<Ruby> for ProcessWeakReferences { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + if crate::mmtk().get_plan().current_gc_may_move_object() { + let gc_tls: &mut GCThreadTLS = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + let visit_object = |_worker, target_object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(), + "Destination is not an MMTk object" + ); + + target_object + .get_forwarded_object() + .unwrap_or(target_object) + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(visit_object, || { + self.process_weak_references(true); + }) + } else { + self.process_weak_references(false); + } + } +} + +impl ProcessWeakReferences { + fn process_weak_references(&mut self, moving_gc: bool) { + let mut weak_references = crate::binding() + .weak_proc + .weak_references + .try_lock() + .expect("Mutators should not be holding the lock."); + + weak_references.retain_mut(|object_ptr| { + let object = object_ptr.get_forwarded_object().unwrap_or(*object_ptr); + + if object != *object_ptr { + *object_ptr = object; + } + + if object.is_reachable() { + (upcalls().handle_weak_references)(object, moving_gc); + + true + } else { + false + } + }); + } +} + +trait GlobalTableProcessingWork { + fn process_table(&mut self); + + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) }; + + // `hash_foreach_replace` depends on `gb_object_moved_p` which has to have the semantics + // of `trace_object` due to the way it is used in `UPDATE_IF_MOVED`. + let forward_object = |_worker, object: ObjectReference, _pin| { + debug_assert!( + mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(), + "{object} is not an MMTk object" + ); + let result = object.forward(); + trace!("Forwarding reference: {object} -> {result}"); + result + }; + + gc_tls + .object_closure + .set_temporarily_and_run_code(forward_object, || { + self.process_table(); + }); + } +} + +struct UpdateFinalizerObjIdTables; +impl GlobalTableProcessingWork for UpdateFinalizerObjIdTables { + fn process_table(&mut self) { + (crate::upcalls().update_finalizer_table)(); + } +} +impl GCWork<Ruby> for UpdateFinalizerObjIdTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateGlobalTables { + idx: i32, +} +impl GlobalTableProcessingWork for UpdateGlobalTables { + fn process_table(&mut self) { + (crate::upcalls().update_global_tables)( + self.idx, + crate::mmtk().get_plan().current_gc_may_move_object(), + ) + } +} +impl GCWork<Ruby> for UpdateGlobalTables { + fn do_work(&mut self, worker: &mut GCWorker<Ruby>, mmtk: &'static mmtk::MMTK<Ruby>) { + GlobalTableProcessingWork::do_work(self, worker, mmtk); + } +} + +struct UpdateWbUnprotectedObjectsList; + +impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList { + fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) { + let mut objects = crate::binding().wb_unprotected_objects.try_lock().expect( + "Someone is holding the lock of wb_unprotected_objects during weak processing phase?", + ); + + let old_objects = std::mem::take(&mut *objects); + + debug!("Updating {} WB-unprotected objects", old_objects.len()); + + for object in old_objects { + if object.is_reachable() { + // Forward and add back to the candidate list. + let new_object = object.forward(); + trace!("Forwarding WB-unprotected object: {object} -> {new_object}"); + objects.insert(new_object); + } else { + trace!("Removing WB-unprotected object from list: {object}"); + } + } + + debug!("Retained {} live WB-unprotected objects.", objects.len()); + } +} + +// Provide a shorthand `object.forward()`. +trait Forwardable { + fn forward(&self) -> Self; +} + +impl Forwardable for ObjectReference { + fn forward(&self) -> Self { + self.get_forwarded_object().unwrap_or(*self) + } +} |