summaryrefslogtreecommitdiff
path: root/gc/mmtk
diff options
context:
space:
mode:
Diffstat (limited to 'gc/mmtk')
-rw-r--r--gc/mmtk/Cargo.lock469
-rw-r--r--gc/mmtk/Cargo.toml9
-rw-r--r--gc/mmtk/cbindgen.toml5
-rw-r--r--gc/mmtk/extconf.rb4
-rw-r--r--gc/mmtk/mmtk.c713
-rw-r--r--gc/mmtk/mmtk.h41
-rw-r--r--gc/mmtk/src/abi.rs102
-rw-r--r--gc/mmtk/src/api.rs322
-rw-r--r--gc/mmtk/src/binding.rs10
-rw-r--r--gc/mmtk/src/collection.rs61
-rw-r--r--gc/mmtk/src/heap/cpu_heap_trigger.rs370
-rw-r--r--gc/mmtk/src/heap/mod.rs9
-rw-r--r--gc/mmtk/src/heap/ruby_heap_trigger.rs105
-rw-r--r--gc/mmtk/src/lib.rs36
-rw-r--r--gc/mmtk/src/object_model.rs53
-rw-r--r--gc/mmtk/src/pinning_registry.rs187
-rw-r--r--gc/mmtk/src/scanning.rs72
-rw-r--r--gc/mmtk/src/utils.rs60
-rw-r--r--gc/mmtk/src/weak_proc.rs257
19 files changed, 2175 insertions, 710 deletions
diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock
index f909c3ae33..910048fa80 100644
--- a/gc/mmtk/Cargo.lock
+++ b/gc/mmtk/Cargo.lock
@@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anstream"
-version = "0.6.14"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -28,35 +28,36 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.7"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
-version = "1.1.0"
+version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
dependencies = [
"windows-sys",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
dependencies = [
"anstyle",
+ "once_cell_polyfill",
"windows-sys",
]
@@ -87,54 +88,54 @@ checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c"
[[package]]
name = "autocfg"
-version = "1.3.0"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
-version = "2.6.0"
+version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
[[package]]
name = "built"
-version = "0.7.3"
+version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6a6c0b39c38fd754ac338b00a88066436389c0f029da5d37d1e01091d9b7c17"
+checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b"
dependencies = [
"git2",
]
[[package]]
name = "bytemuck"
-version = "1.16.1"
+version = "1.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
+checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c"
dependencies = [
"bytemuck_derive",
]
[[package]]
name = "bytemuck_derive"
-version = "1.7.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b"
+checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "cc"
-version = "1.0.100"
+version = "1.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b"
+checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
dependencies = [
"jobserver",
"libc",
- "once_cell",
+ "shlex",
]
[[package]]
@@ -145,15 +146,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "colorchoice"
-version = "1.0.1"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "crossbeam"
@@ -170,18 +171,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.13"
+version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -198,41 +199,41 @@ dependencies = [
[[package]]
name = "crossbeam-queue"
-version = "0.3.11"
+version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
+checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
-version = "0.8.20"
+version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "delegate"
-version = "0.12.0"
+version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b"
+checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "downcast-rs"
-version = "1.2.1"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf"
[[package]]
name = "either"
-version = "1.12.0"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "enum-map"
@@ -251,14 +252,14 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "env_filter"
-version = "0.1.0"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
dependencies = [
"log",
"regex",
@@ -266,14 +267,14 @@ dependencies = [
[[package]]
name = "env_logger"
-version = "0.11.3"
+version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9"
+checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f"
dependencies = [
"anstream",
"anstyle",
"env_filter",
- "humantime",
+ "jiff",
"log",
]
@@ -287,10 +288,22 @@ dependencies = [
]
[[package]]
+name = "getrandom"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi",
+]
+
+[[package]]
name = "git2"
-version = "0.18.3"
+version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70"
+checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
dependencies = [
"bitflags",
"libc",
@@ -312,53 +325,99 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]]
-name = "humantime"
-version = "2.1.0"
+name = "hermit-abi"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
[[package]]
name = "idna"
-version = "0.5.0"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8"
+dependencies = [
+ "idna_mapping",
"unicode-bidi",
"unicode-normalization",
]
[[package]]
+name = "idna_mapping"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792"
+dependencies = [
+ "unicode-joining-type",
+]
+
+[[package]]
name = "is-terminal"
-version = "0.4.12"
+version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.5.1",
"libc",
"windows-sys",
]
[[package]]
name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itertools"
-version = "0.12.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
[[package]]
+name = "jiff"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"
+dependencies = [
+ "jiff-static",
+ "log",
+ "portable-atomic",
+ "portable-atomic-util",
+ "serde",
+]
+
+[[package]]
+name = "jiff-static"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
+[[package]]
name = "jobserver"
-version = "0.1.31"
+version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
dependencies = [
+ "getrandom",
"libc",
]
@@ -370,15 +429,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.155"
+version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]]
name = "libgit2-sys"
-version = "0.16.2+1.7.2"
+version = "0.18.3+1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8"
+checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
dependencies = [
"cc",
"libc",
@@ -388,9 +447,9 @@ dependencies = [
[[package]]
name = "libz-sys"
-version = "1.1.18"
+version = "1.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e"
+checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
dependencies = [
"cc",
"libc",
@@ -410,9 +469,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.21"
+version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "memchr"
@@ -431,20 +490,22 @@ dependencies = [
[[package]]
name = "mmtk"
-version = "0.30.0"
-source = "git+https://github.com/mmtk/mmtk-core.git?rev=051bc7470feef915c445305301e6113f86d3957b#051bc7470feef915c445305301e6113f86d3957b"
+version = "0.31.0"
+source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
dependencies = [
"atomic",
"atomic-traits",
"atomic_refcell",
"built",
"bytemuck",
+ "bytemuck_derive",
"cfg-if",
"crossbeam",
"delegate",
"downcast-rs",
"enum-map",
"env_logger",
+ "idna_adapter",
"is-terminal",
"itertools",
"lazy_static",
@@ -456,24 +517,25 @@ dependencies = [
"num_cpus",
"portable-atomic",
"probe",
+ "rayon-core",
"regex",
"rustversion",
"spin",
"static_assertions",
"strum",
"strum_macros",
- "sysinfo 0.30.12",
+ "sysinfo 0.33.1",
]
[[package]]
name = "mmtk-macros"
-version = "0.30.0"
-source = "git+https://github.com/mmtk/mmtk-core.git?rev=051bc7470feef915c445305301e6113f86d3957b#051bc7470feef915c445305301e6113f86d3957b"
+version = "0.31.0"
+source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -487,7 +549,7 @@ dependencies = [
"mmtk",
"once_cell",
"probe",
- "sysinfo 0.32.0",
+ "sysinfo 0.32.1",
]
[[package]]
@@ -514,15 +576,21 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.9",
"libc",
]
[[package]]
name = "once_cell"
-version = "1.19.0"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "percent-encoding"
@@ -532,15 +600,24 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pkg-config"
-version = "0.3.30"
+version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "portable-atomic"
-version = "1.6.0"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
+checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+dependencies = [
+ "portable-atomic",
+]
[[package]]
name = "probe"
@@ -574,23 +651,29 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
-version = "1.0.36"
+version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
+[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -612,9 +695,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.10.5"
+version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
@@ -624,9 +707,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.7"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
@@ -635,24 +718,24 @@ dependencies = [
[[package]]
name = "regex-syntax"
-version = "0.8.4"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustc_version"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustversion"
-version = "1.0.17"
+version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
+checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
[[package]]
name = "scopeguard"
@@ -662,9 +745,41 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
-version = "1.0.23"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
+[[package]]
+name = "serde"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "smallvec"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
[[package]]
name = "spin"
@@ -683,21 +798,21 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strum"
-version = "0.26.3"
+version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32"
[[package]]
name = "strum_macros"
-version = "0.26.4"
+version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -712,9 +827,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.68"
+version = "2.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
+checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
dependencies = [
"proc-macro2",
"quote",
@@ -723,38 +838,37 @@ dependencies = [
[[package]]
name = "sysinfo"
-version = "0.30.12"
+version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
+checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af"
dependencies = [
- "cfg-if",
"core-foundation-sys",
"libc",
+ "memchr",
"ntapi",
- "once_cell",
"rayon",
- "windows 0.52.0",
+ "windows",
]
[[package]]
name = "sysinfo"
-version = "0.32.0"
+version = "0.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791"
+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
- "windows 0.57.0",
+ "windows",
]
[[package]]
name = "tinyvec"
-version = "1.6.0"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
dependencies = [
"tinyvec_macros",
]
@@ -767,30 +881,36 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicode-bidi"
-version = "0.3.15"
+version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
[[package]]
name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
+
+[[package]]
+name = "unicode-joining-type"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180"
[[package]]
name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
dependencies = [
"tinyvec",
]
[[package]]
name = "url"
-version = "2.5.2"
+version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [
"form_urlencoded",
"idna",
@@ -798,6 +918,12 @@ dependencies = [
]
[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
+[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -811,9 +937,18 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.14.2+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
[[package]]
name = "winapi"
@@ -839,30 +974,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
-dependencies = [
- "windows-core 0.52.0",
- "windows-targets",
-]
-
-[[package]]
-name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
- "windows-core 0.57.0",
- "windows-targets",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
+ "windows-core",
"windows-targets",
]
@@ -886,7 +1002,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -897,7 +1013,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -911,18 +1027,18 @@ dependencies = [
[[package]]
name = "windows-sys"
-version = "0.52.0"
+version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
@@ -936,48 +1052,57 @@ dependencies = [
[[package]]
name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags",
+]
diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml
index 66af77fe61..d856122900 100644
--- a/gc/mmtk/Cargo.toml
+++ b/gc/mmtk/Cargo.toml
@@ -21,11 +21,11 @@ probe = "0.5"
sysinfo = "0.32.0"
[dependencies.mmtk]
-features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery", "immix_non_moving"]
+features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"]
# Uncomment the following lines to use mmtk-core from the official repository.
git = "https://github.com/mmtk/mmtk-core.git"
-rev = "051bc7470feef915c445305301e6113f86d3957b"
+rev = "c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
# Uncomment the following line to use mmtk-core from a local repository.
# path = "../../../mmtk-core"
@@ -35,3 +35,8 @@ default = []
# When moving an object, clear its original copy.
clear_old_copy = []
+
+# Enable extra assertions in release build. For debugging.
+extra_assert = []
+
+[workspace]
diff --git a/gc/mmtk/cbindgen.toml b/gc/mmtk/cbindgen.toml
index c66f829b3d..b99c30efc8 100644
--- a/gc/mmtk/cbindgen.toml
+++ b/gc/mmtk/cbindgen.toml
@@ -20,6 +20,11 @@ typedef void *MMTk_Address;
typedef void *MMTk_ObjectReference;
typedef void *MMTk_NullableObjectReference;
typedef uint32_t MMTk_AllocationSemantics;
+
+typedef struct MMTk_BumpPointer {
+ uintptr_t cursor;
+ uintptr_t limit;
+} MMTk_BumpPointer;
"""
[export]
diff --git a/gc/mmtk/extconf.rb b/gc/mmtk/extconf.rb
index 5f4228972b..c0e788037e 100644
--- a/gc/mmtk/extconf.rb
+++ b/gc/mmtk/extconf.rb
@@ -15,6 +15,10 @@ create_gc_makefile("mmtk") do |makefile|
MMTK_BUILD = debug
LIBMMTK_RUBY = libmmtk_ruby.#$LIBEXT
RUSTSRCS = #{rustsrcs.join(" \\\n\t ")}
+
+ ifeq ($(MMTK_BUILD), debug)
+ CPPFLAGS += -DMMTK_DEBUG
+ endif
MAKEFILE
]
end
diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c
index f52b183fa7..95176b692b 100644
--- a/gc/mmtk/mmtk.c
+++ b/gc/mmtk/mmtk.c
@@ -21,13 +21,10 @@ struct objspace {
bool gc_stress;
size_t gc_count;
+ size_t moving_gc_count;
size_t total_gc_time;
size_t total_allocated_objects;
- st_table *id_to_obj_tbl;
- st_table *obj_to_id_tbl;
- unsigned long long next_object_id;
-
st_table *finalizer_table;
struct MMTk_final_job *finalizer_jobs;
rb_postponed_job_handle_t finalizer_postponed_job;
@@ -36,19 +33,38 @@ struct objspace {
unsigned long live_ractor_cache_count;
pthread_mutex_t mutex;
+ rb_atomic_t mutator_blocking_count;
bool world_stopped;
pthread_cond_t cond_world_stopped;
pthread_cond_t cond_world_started;
size_t start_the_world_count;
+ pthread_mutex_t event_hook_mutex;
+
+ struct {
+ bool gc_thread_crashed;
+ char crash_msg[256];
+ } crash_context;
+
struct rb_gc_vm_context vm_context;
+
+ unsigned int fork_hook_vm_lock_lev;
};
+#define OBJ_FREE_BUF_CAPACITY 128
+
struct MMTk_ractor_cache {
struct ccan_list_node list_node;
MMTk_Mutator *mutator;
bool gc_mutator_p;
+
+ MMTk_BumpPointer *bump_pointer;
+
+ MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY];
+ size_t obj_free_parallel_count;
+ MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY];
+ size_t obj_free_non_parallel_count;
};
struct MMTk_final_job {
@@ -63,7 +79,7 @@ struct MMTk_final_job {
void *data;
} dfree;
struct {
- VALUE object_id;
+ /* HACK: we store the object ID on the 0th element of this array. */
VALUE finalizer_array;
} finalize;
} as;
@@ -71,12 +87,22 @@ struct MMTk_final_job {
#ifdef RB_THREAD_LOCAL_SPECIFIER
RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls;
+
+RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object;
#else
# error We currently need language-supported TLS
#endif
+#ifdef MMTK_DEBUG
+# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__))
+#else
+# define MMTK_ASSERT(expr, ...) ((void)0)
+#endif
+
#include <pthread.h>
+static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
+
static void
rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls)
{
@@ -109,7 +135,7 @@ rb_mmtk_stop_the_world(void)
}
static void
-rb_mmtk_resume_mutators(void)
+rb_mmtk_resume_mutators(bool current_gc_may_move)
{
struct objspace *objspace = rb_gc_get_objspace();
@@ -120,6 +146,7 @@ rb_mmtk_resume_mutators(void)
objspace->world_stopped = false;
objspace->gc_count++;
+ if (current_gc_may_move) objspace->moving_gc_count++;
pthread_cond_broadcast(&objspace->cond_world_started);
if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
@@ -127,13 +154,17 @@ rb_mmtk_resume_mutators(void)
}
}
+static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache);
+
static void
rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
{
struct objspace *objspace = rb_gc_get_objspace();
size_t starting_gc_count = objspace->gc_count;
- int lock_lev = rb_gc_vm_lock();
+ RUBY_ATOMIC_INC(objspace->mutator_blocking_count);
+ int lock_lev = RB_GC_VM_LOCK();
+ RUBY_ATOMIC_DEC(objspace->mutator_blocking_count);
int err;
if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) {
rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err));
@@ -155,6 +186,11 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
rb_gc_vm_barrier();
+ struct MMTk_ractor_cache *rc;
+ ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
+ mmtk_flush_obj_free_buffer(rc);
+ }
+
objspace->world_stopped = true;
pthread_cond_broadcast(&objspace->cond_world_stopped);
@@ -164,6 +200,10 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex);
}
+ if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) {
+ rb_bug("%s", objspace->crash_context.crash_msg);
+ }
+
if (objspace->measure_gc_time) {
struct timespec gc_end_time;
clock_gettime(CLOCK_MONOTONIC, &gc_end_time);
@@ -177,7 +217,19 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err));
}
- rb_gc_vm_unlock(lock_lev);
+ RB_GC_VM_UNLOCK(lock_lev);
+}
+
+static void
+rb_mmtk_before_updating_jit_code(void)
+{
+ rb_gc_before_updating_jit_code();
+}
+
+static void
+rb_mmtk_after_updating_jit_code(void)
+{
+ rb_gc_after_updating_jit_code();
}
static size_t
@@ -191,9 +243,7 @@ static void
rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data)
{
struct objspace *objspace = rb_gc_get_objspace();
-
struct MMTk_ractor_cache *ractor_cache;
- RUBY_ASSERT(ractor_cache != NULL);
ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) {
visit_mutator(ractor_cache->mutator, data);
@@ -203,7 +253,9 @@ rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), v
static void
rb_mmtk_scan_gc_roots(void)
{
- // rb_gc_mark_roots(rb_gc_get_objspace(), NULL);
+ struct objspace *objspace = rb_gc_get_objspace();
+
+ rb_gc_mark_roots(objspace, NULL);
}
static int
@@ -223,15 +275,12 @@ rb_mmtk_scan_objspace(void)
st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace);
}
- st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace);
-
struct MMTk_final_job *job = objspace->finalizer_jobs;
while (job != NULL) {
switch (job->kind) {
case MMTK_FINAL_JOB_DFREE:
break;
case MMTK_FINAL_JOB_FINALIZE:
- rb_gc_impl_mark(objspace, job->as.finalize.object_id);
rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array);
break;
default:
@@ -243,27 +292,45 @@ rb_mmtk_scan_objspace(void)
}
static void
-rb_mmtk_scan_roots_in_mutator_thread(MMTk_VMMutatorThread mutator, MMTk_VMWorkerThread worker)
+rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to)
+{
+ rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to);
+}
+
+static void
+rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object)
{
- if (mutator->gc_mutator_p) {
- struct objspace *objspace = rb_gc_get_objspace();
+ VALUE object = (VALUE)mmtk_object;
- rb_gc_worker_thread_set_vm_context(&objspace->vm_context);
- rb_gc_mark_roots(objspace, NULL);
- rb_gc_worker_thread_unset_vm_context(&objspace->vm_context);
+ if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) {
+ marking_parent_object = object;
+ rb_gc_update_object_references(rb_gc_get_objspace(), object);
+ marking_parent_object = 0;
}
}
static void
-rb_mmtk_scan_object_ruby_style(MMTk_ObjectReference object)
+rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
{
+ marking_parent_object = (VALUE)object;
rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
+ marking_parent_object = 0;
}
static void
-rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
+rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving)
{
- rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
+ VALUE object = (VALUE)mmtk_object;
+
+ marking_parent_object = object;
+
+ rb_gc_handle_weak_references(object);
+
+ if (moving) {
+ rb_gc_update_object_references(rb_gc_get_objspace(), object);
+ }
+
+ marking_parent_object = 0;
}
static void
@@ -273,12 +340,16 @@ rb_mmtk_call_obj_free(MMTk_ObjectReference object)
struct objspace *objspace = rb_gc_get_objspace();
if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) {
- rb_gc_worker_thread_set_vm_context(&objspace->vm_context);
+ pthread_mutex_lock(&objspace->event_hook_mutex);
rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ);
- rb_gc_worker_thread_unset_vm_context(&objspace->vm_context);
+ pthread_mutex_unlock(&objspace->event_hook_mutex);
}
rb_gc_obj_free(objspace, obj);
+
+#ifdef MMTK_DEBUG
+ memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj));
+#endif
}
static size_t
@@ -290,31 +361,34 @@ rb_mmtk_vm_live_bytes(void)
static void
make_final_job(struct objspace *objspace, VALUE obj, VALUE table)
{
- RUBY_ASSERT(RB_FL_TEST(obj, RUBY_FL_FINALIZE));
- RUBY_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)table));
- RUBY_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY);
-
- RB_FL_UNSET(obj, RUBY_FL_FINALIZE);
+ MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY);
struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job));
job->next = objspace->finalizer_jobs;
job->kind = MMTK_FINAL_JOB_FINALIZE;
- job->as.finalize.object_id = rb_obj_id((VALUE)obj);
job->as.finalize.finalizer_array = table;
objspace->finalizer_jobs = job;
}
static int
-rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data)
+rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error)
{
- RUBY_ASSERT(RB_FL_TEST(key, RUBY_FL_FINALIZE));
- RUBY_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value));
- RUBY_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY);
+ MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value));
+ MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY);
struct objspace *objspace = (struct objspace *)data;
- if (!mmtk_is_reachable((MMTk_ObjectReference)key)) {
+ if (mmtk_is_reachable((MMTk_ObjectReference)key)) {
+ VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false);
+
+ MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE));
+
+ if (new_key_location != key) {
+ return ST_REPLACE;
+ }
+ }
+ else {
make_final_job(objspace, (VALUE)key, (VALUE)value);
rb_postponed_job_trigger(objspace->finalizer_postponed_job);
@@ -325,70 +399,119 @@ rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data)
return ST_CONTINUE;
}
+static int
+rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing)
+{
+ *key = rb_mmtk_call_object_closure((VALUE)*key, false);
+
+ return ST_CONTINUE;
+}
+
static void
rb_mmtk_update_finalizer_table(void)
{
struct objspace *objspace = rb_gc_get_objspace();
- // TODO: replace with st_foreach_with_replace when GC is moving
- st_foreach(objspace->finalizer_table, rb_mmtk_update_finalizer_table_i, (st_data_t)objspace);
+ st_foreach_with_replace(
+ objspace->finalizer_table,
+ rb_mmtk_update_finalizer_table_i,
+ rb_mmtk_update_finalizer_table_replace_i,
+ (st_data_t)objspace
+ );
}
static int
-rb_mmtk_update_table_i(VALUE val, void *data)
+rb_mmtk_global_tables_count(void)
+{
+ return RB_GC_VM_WEAK_TABLE_COUNT;
+}
+
+static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
+
+static int
+rb_mmtk_update_global_tables_i(VALUE val, void *data)
{
if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
return ST_DELETE;
}
+ // TODO: check only if in moving GC
+ if (rb_mmtk_call_object_closure(val, false) != val) {
+ return ST_REPLACE;
+ }
+
return ST_CONTINUE;
}
static int
-rb_mmtk_update_obj_id_tables_obj_to_id_i(st_data_t key, st_data_t val, st_data_t data)
+rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data)
{
- RUBY_ASSERT(RB_FL_TEST(key, FL_SEEN_OBJ_ID));
-
- if (!mmtk_is_reachable((MMTk_ObjectReference)key)) {
- return ST_DELETE;
- }
+ // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice
+ *ptr = rb_mmtk_call_object_closure(*ptr, false);
return ST_CONTINUE;
}
-static int
-rb_mmtk_update_obj_id_tables_id_to_obj_i(st_data_t key, st_data_t val, st_data_t data)
+static void
+rb_mmtk_update_global_tables(int table, bool moving)
{
- RUBY_ASSERT(RB_FL_TEST(val, FL_SEEN_OBJ_ID));
+ MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
- if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
- return ST_DELETE;
- }
+ rb_gc_vm_weak_table_foreach(
+ rb_mmtk_update_global_tables_i,
+ rb_mmtk_update_global_tables_replace_i,
+ NULL,
+ !moving,
+ (enum rb_gc_vm_weak_tables)table
+ );
+}
- return ST_CONTINUE;
+static bool
+rb_mmtk_special_const_p(MMTk_ObjectReference object)
+{
+ VALUE obj = (VALUE)object;
+
+ return RB_SPECIAL_CONST_P(obj);
}
+RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2)
+RBIMPL_ATTR_NORETURN()
static void
-rb_mmtk_update_obj_id_tables(void)
+rb_mmtk_gc_thread_bug(const char *msg, ...)
{
struct objspace *objspace = rb_gc_get_objspace();
- st_foreach(objspace->obj_to_id_tbl, rb_mmtk_update_obj_id_tables_obj_to_id_i, 0);
- st_foreach(objspace->id_to_obj_tbl, rb_mmtk_update_obj_id_tables_id_to_obj_i, 0);
+ objspace->crash_context.gc_thread_crashed = true;
+
+ va_list args;
+ va_start(args, msg);
+ vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args);
+ va_end(args);
+
+ fprintf(stderr, "-- GC thread backtrace "
+ "-------------------------------------------\n");
+ rb_gc_print_backtrace();
+ fprintf(stderr, "\n");
+
+ rb_mmtk_resume_mutators(false);
+
+ sleep(5);
+
+ rb_bug("rb_mmtk_gc_thread_bug");
}
-static int
-rb_mmtk_global_tables_count(void)
+RBIMPL_ATTR_NORETURN()
+static void
+rb_mmtk_gc_thread_panic_handler(void)
{
- return RB_GC_VM_WEAK_TABLE_COUNT;
+ rb_mmtk_gc_thread_bug("MMTk GC thread panicked");
}
+RBIMPL_ATTR_NORETURN()
static void
-rb_mmtk_update_global_tables(int table)
+rb_mmtk_mutator_thread_panic_handler(void)
{
- RUBY_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
-
- rb_gc_vm_weak_table_foreach(rb_mmtk_update_table_i, NULL, NULL, true, (enum rb_gc_vm_weak_tables)table);
+ rb_bug("Ruby mutator thread panicked");
}
// Bootup
@@ -398,19 +521,24 @@ MMTk_RubyUpcalls ruby_upcalls = {
rb_mmtk_stop_the_world,
rb_mmtk_resume_mutators,
rb_mmtk_block_for_gc,
+ rb_mmtk_before_updating_jit_code,
+ rb_mmtk_after_updating_jit_code,
rb_mmtk_number_of_mutators,
rb_mmtk_get_mutators,
rb_mmtk_scan_gc_roots,
rb_mmtk_scan_objspace,
- rb_mmtk_scan_roots_in_mutator_thread,
- rb_mmtk_scan_object_ruby_style,
+ rb_mmtk_move_obj_during_marking,
+ rb_mmtk_update_object_references,
rb_mmtk_call_gc_mark_children,
+ rb_mmtk_handle_weak_references,
rb_mmtk_call_obj_free,
rb_mmtk_vm_live_bytes,
rb_mmtk_update_global_tables,
rb_mmtk_global_tables_count,
rb_mmtk_update_finalizer_table,
- rb_mmtk_update_obj_id_tables,
+ rb_mmtk_special_const_p,
+ rb_mmtk_mutator_thread_panic_handler,
+ rb_mmtk_gc_thread_panic_handler,
};
// Use max 80% of the available memory by default for MMTk
@@ -434,12 +562,14 @@ void *
rb_gc_impl_objspace_alloc(void)
{
MMTk_Builder *builder = rb_mmtk_builder_init();
- mmtk_init_binding(builder, NULL, &ruby_upcalls, (MMTk_ObjectReference)Qundef);
+ MMTk_RubyBindingOptions binding_options = {
+ .suffix_size = RB_GC_OBJ_SUFFIX_SIZE,
+ };
+ mmtk_init_binding(builder, &binding_options, &ruby_upcalls);
return calloc(1, sizeof(struct objspace));
}
-static void objspace_obj_id_init(struct objspace *objspace);
static void gc_run_finalizers(void *data);
void
@@ -449,8 +579,6 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
objspace->measure_gc_time = true;
- objspace_obj_id_init(objspace);
-
objspace->finalizer_table = st_init_numtable();
objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace);
@@ -459,6 +587,8 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
+
+ objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
}
void
@@ -476,10 +606,11 @@ rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
}
objspace->live_ractor_cache_count++;
- struct MMTk_ractor_cache *cache = malloc(sizeof(struct MMTk_ractor_cache));
+ struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache));
ccan_list_add(&objspace->ractor_caches, &cache->list_node);
cache->mutator = mmtk_bind_mutator(cache);
+ cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator);
return cache;
}
@@ -492,7 +623,15 @@ rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr)
ccan_list_del(&cache->list_node);
- RUBY_ASSERT(objspace->live_ractor_cache_count > 1);
+ mmtk_flush_obj_free_buffer(cache);
+
+ if (ruby_free_at_exit_p()) {
+ MMTK_ASSERT(objspace->live_ractor_cache_count > 0);
+ }
+ else {
+ MMTK_ASSERT(objspace->live_ractor_cache_count > 1);
+ }
+
objspace->live_ractor_cache_count--;
mmtk_destroy_mutator(cache->mutator);
@@ -502,15 +641,31 @@ void rb_gc_impl_set_params(void *objspace_ptr) { }
static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; }
+#if SIZEOF_VALUE >= 8
+#define MMTK_HEAP_COUNT 12
+#define MMTK_MAX_OBJ_SIZE 1024
+static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
+ 32, 40, 64, 80, 96, 128, 160, 256, 512, 640, 768, MMTK_MAX_OBJ_SIZE, 0
+};
+#else
+#define MMTK_HEAP_COUNT 5
+#define MMTK_MAX_OBJ_SIZE 512
+static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
+ 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0
+};
+#endif
+
void
rb_gc_impl_init(void)
{
VALUE gc_constants = rb_hash_new();
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(640));
- // Pretend we have 5 size pools
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(5));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT));
+ // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0));
OBJ_FREEZE(gc_constants);
rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
@@ -524,10 +679,6 @@ rb_gc_impl_init(void)
rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
-static size_t heap_sizes[6] = {
- 40, 80, 160, 320, 640, 0
-};
-
size_t *
rb_gc_impl_heap_sizes(void *objspace_ptr)
{
@@ -568,8 +719,8 @@ rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool i
bool
rb_gc_impl_during_gc_p(void *objspace_ptr)
{
- // TODO
- return false;
+ struct objspace *objspace = objspace_ptr;
+ return objspace->world_stopped;
}
static void
@@ -639,17 +790,106 @@ rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
// TODO
}
+struct rb_gc_vm_context *
+rb_gc_impl_get_vm_context(void *objspace_ptr)
+{
+ struct objspace *objspace = objspace_ptr;
+
+ return &objspace->vm_context;
+}
+
// Object allocation
+static VALUE
+rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align)
+{
+ MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer;
+ if (bump_pointer == NULL) return 0;
+
+ uintptr_t cursor = bump_pointer->cursor;
+
+ // Ensure cursor is aligned
+ size_t mask = align - 1;
+ cursor = (cursor + mask) & ~mask;
+
+ cursor += size;
+
+ if (cursor > bump_pointer->limit) {
+ return 0;
+ }
+ else {
+ VALUE obj = cursor - size;
+ bump_pointer->cursor = cursor;
+ return obj;
+ }
+}
+
+static bool
+obj_can_parallel_free_p(VALUE obj)
+{
+ switch (RB_BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+ case T_BIGNUM:
+ case T_COMPLEX:
+ case T_FLOAT:
+ case T_HASH:
+ case T_OBJECT:
+ case T_RATIONAL:
+ case T_REGEXP:
+ case T_STRING:
+ case T_STRUCT:
+ case T_SYMBOL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void
+mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache)
+{
+ if (cache->obj_free_parallel_count > 0) {
+ mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
+ cache->obj_free_parallel_count, true);
+ cache->obj_free_parallel_count = 0;
+ }
+ if (cache->obj_free_non_parallel_count > 0) {
+ mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
+ cache->obj_free_non_parallel_count, false);
+ cache->obj_free_non_parallel_count = 0;
+ }
+}
+
+static inline void
+mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj)
+{
+ if (obj_can_parallel_free_p(obj)) {
+ cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj;
+ if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
+ mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
+ cache->obj_free_parallel_count, true);
+ cache->obj_free_parallel_count = 0;
+ }
+ }
+ else {
+ cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj;
+ if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
+ mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
+ cache->obj_free_non_parallel_count, false);
+ cache->obj_free_non_parallel_count = 0;
+ }
+ }
+}
+
VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
{
#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0
struct objspace *objspace = objspace_ptr;
struct MMTk_ractor_cache *ractor_cache = cache_ptr;
- if (alloc_size > 640) rb_bug("too big");
- for (int i = 0; i < 5; i++) {
+ if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big");
+ for (int i = 0; i < MMTK_HEAP_COUNT; i++) {
if (alloc_size == heap_sizes[i]) break;
if (alloc_size < heap_sizes[i]) {
alloc_size = heap_sizes[i];
@@ -661,19 +901,24 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
mmtk_handle_user_collection_request(ractor_cache, false, false);
}
- VALUE *alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size + 8, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)][suffix (RB_GC_OBJ_SUFFIX_SIZE)]
+ alloc_size += sizeof(VALUE) + RB_GC_OBJ_SUFFIX_SIZE;
+
+ VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN);
+ if (!alloc_obj) {
+ alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ }
+
alloc_obj++;
- alloc_obj[-1] = alloc_size;
+ alloc_obj[-1] = alloc_size - sizeof(VALUE) - RB_GC_OBJ_SUFFIX_SIZE;
alloc_obj[0] = flags;
alloc_obj[1] = klass;
- if (alloc_size > 16) alloc_obj[2] = v1;
- if (alloc_size > 24) alloc_obj[3] = v2;
- if (alloc_size > 32) alloc_obj[4] = v3;
- mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size + 8, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ // TODO: implement fast path for mmtk_post_alloc
+ mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
// TODO: only add when object needs obj_free to be called
- mmtk_add_obj_free_candidate(alloc_obj);
+ mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj);
objspace->total_allocated_objects++;
@@ -689,7 +934,7 @@ rb_gc_impl_obj_slot_size(VALUE obj)
size_t
rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
{
- for (int i = 0; i < 5; i++) {
+ for (int i = 0; i < MMTK_HEAP_COUNT; i++) {
if (size == heap_sizes[i]) return i;
if (size < heap_sizes[i]) return i;
}
@@ -700,26 +945,26 @@ rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
- return size <= 640;
+ return size <= MMTK_MAX_OBJ_SIZE;
}
// Malloc
void *
-rb_gc_impl_malloc(void *objspace_ptr, size_t size)
+rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
// TODO: don't use system malloc
return malloc(size);
}
void *
-rb_gc_impl_calloc(void *objspace_ptr, size_t size)
+rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
// TODO: don't use system calloc
return calloc(1, size);
}
void *
-rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size)
+rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
// TODO: don't use system realloc
return realloc(ptr, new_size);
@@ -735,15 +980,34 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { }
// Marking
+static inline VALUE
+rb_mmtk_call_object_closure(VALUE obj, bool pin)
+{
+ if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) {
+ enum { info_size = 256 };
+ char obj_info_buf[info_size];
+ rb_raw_obj_info(obj_info_buf, info_size, obj);
+
+ char parent_obj_info_buf[info_size];
+ rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object);
+
+ rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
+ }
+
+ return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function(
+ rb_mmtk_gc_thread_tls->object_closure.rust_closure,
+ rb_mmtk_gc_thread_tls->gc_context,
+ (MMTk_ObjectReference)obj,
+ pin
+ );
+}
+
void
rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) return;
- rb_mmtk_gc_thread_tls->object_closure.c_function(rb_mmtk_gc_thread_tls->object_closure.rust_closure,
- rb_mmtk_gc_thread_tls->gc_context,
- (MMTk_ObjectReference)obj,
- false);
+ rb_mmtk_call_object_closure(obj, false);
}
void
@@ -751,8 +1015,10 @@ rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
{
if (RB_SPECIAL_CONST_P(*ptr)) return;
- // TODO: make it movable
- rb_gc_impl_mark(objspace_ptr, *ptr);
+ VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false);
+ if (new_obj != *ptr) {
+ *ptr = new_obj;
+ }
}
void
@@ -760,8 +1026,7 @@ rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) return;
- // TODO: also pin
- rb_gc_impl_mark(objspace_ptr, obj);
+ rb_mmtk_call_object_closure(obj, true);
}
void
@@ -773,28 +1038,35 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
}
void
-rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr)
+rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
- mmtk_mark_weak((MMTk_ObjectReference *)ptr);
+ RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE);
+ mmtk_declare_weak_references((MMTk_ObjectReference)obj);
}
-void
-rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr)
+bool
+rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
{
- mmtk_remove_weak((MMTk_ObjectReference *)ptr);
+ return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj);
}
// Compaction
+void
+rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
+{
+ mmtk_register_pinning_obj((MMTk_ObjectReference)obj);
+}
+
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
- rb_bug("unimplemented");
+ return rb_mmtk_call_object_closure(obj, false) != obj;
}
VALUE
-rb_gc_impl_location(void *objspace_ptr, VALUE value)
+rb_gc_impl_location(void *objspace_ptr, VALUE obj)
{
- rb_bug("unimplemented");
+ return rb_mmtk_call_object_closure(obj, false);
}
// Write barriers
@@ -803,6 +1075,23 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache();
+ if (SPECIAL_CONST_P(b)) return;
+
+#ifdef MMTK_DEBUG
+ if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)a)) {
+ char buff[256];
+ rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a));
+ }
+
+ if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)b)) {
+ char buff[256];
+ rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b));
+ }
+#endif
+
+ MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE);
+ MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE);
+
mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a);
}
@@ -907,7 +1196,7 @@ gc_run_finalizers_get_final(long i, void *data)
{
VALUE table = (VALUE)data;
- return RARRAY_AREF(table, i);
+ return RARRAY_AREF(table, i + 1);
}
static void
@@ -926,17 +1215,15 @@ gc_run_finalizers(void *data)
job->as.dfree.func(job->as.dfree.data);
break;
case MMTK_FINAL_JOB_FINALIZE: {
- VALUE object_id = job->as.finalize.object_id;
VALUE finalizer_array = job->as.finalize.finalizer_array;
rb_gc_run_obj_finalizer(
- job->as.finalize.object_id,
- RARRAY_LEN(finalizer_array),
+ RARRAY_AREF(finalizer_array, 0),
+ RARRAY_LEN(finalizer_array) - 1,
gc_run_finalizers_get_final,
(void *)finalizer_array
);
- RB_GC_GUARD(object_id);
RB_GC_GUARD(finalizer_array);
break;
}
@@ -980,6 +1267,8 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
RBASIC(obj)->flags |= FL_FINALIZE;
+ int lev = RB_GC_VM_LOCK();
+
if (st_lookup(objspace->finalizer_table, obj, &data)) {
table = (VALUE)data;
@@ -991,6 +1280,7 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
for (i = 0; i < len; i++) {
VALUE recv = RARRAY_AREF(table, i);
if (rb_equal(recv, block)) {
+ RB_GC_VM_UNLOCK(lev);
return recv;
}
}
@@ -999,11 +1289,13 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
rb_ary_push(table, block);
}
else {
- table = rb_ary_new3(1, block);
+ table = rb_ary_new3(2, rb_obj_id(obj), block);
rb_obj_hide(table);
st_add_direct(objspace->finalizer_table, obj, table);
}
+ RB_GC_VM_UNLOCK(lev);
+
return block;
}
@@ -1013,7 +1305,11 @@ rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
struct objspace *objspace = objspace_ptr;
st_data_t data = obj;
+
+ int lev = RB_GC_VM_LOCK();
st_delete(objspace->finalizer_table, &data, 0);
+ RB_GC_VM_UNLOCK(lev);
+
FL_UNSET(obj, FL_FINALIZE);
}
@@ -1026,14 +1322,17 @@ rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
if (!FL_TEST(obj, FL_FINALIZE)) return;
+ int lev = RB_GC_VM_LOCK();
if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) {
- table = (VALUE)data;
+ table = rb_ary_dup((VALUE)data);
+ RARRAY_ASET(table, 0, rb_obj_id(dest));
st_insert(objspace->finalizer_table, dest, table);
FL_SET(dest, FL_FINALIZE);
}
else {
rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
}
+ RB_GC_VM_UNLOCK(lev);
}
static int
@@ -1057,112 +1356,67 @@ rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
gc_run_finalizers(objspace);
}
- struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
- for (size_t i = 0; i < registered_candidates.len; i++) {
- VALUE obj = (VALUE)registered_candidates.ptr[i];
-
- if (rb_gc_shutdown_call_finalizer_p(obj)) {
- rb_gc_obj_free(objspace_ptr, obj);
- RBASIC(obj)->flags = 0;
+ unsigned int lev = RB_GC_VM_LOCK();
+ {
+ struct MMTk_ractor_cache *rc;
+ ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
+ mmtk_flush_obj_free_buffer(rc);
}
- }
- mmtk_free_raw_vec_of_obj_ref(registered_candidates);
- gc_run_finalizers(objspace);
-}
+ struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
+ for (size_t i = 0; i < registered_candidates.len; i++) {
+ VALUE obj = (VALUE)registered_candidates.ptr[i];
-// Object ID
-static int
-object_id_cmp(st_data_t x, st_data_t y)
-{
- if (RB_TYPE_P(x, T_BIGNUM)) {
- return !rb_big_eql(x, y);
- }
- else {
- return x != y;
+ if (rb_gc_shutdown_call_finalizer_p(obj)) {
+ rb_gc_obj_free(objspace_ptr, obj);
+ RBASIC(obj)->flags = 0;
+ }
+ }
+ mmtk_free_raw_vec_of_obj_ref(registered_candidates);
}
-}
-
-static st_index_t
-object_id_hash(st_data_t n)
-{
- return FIX2LONG(rb_hash((VALUE)n));
-}
-
-#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
-#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
-
-static const struct st_hash_type object_id_hash_type = {
- object_id_cmp,
- object_id_hash,
-};
+ RB_GC_VM_UNLOCK(lev);
-static void
-objspace_obj_id_init(struct objspace *objspace)
-{
- objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
- objspace->obj_to_id_tbl = st_init_numtable();
- objspace->next_object_id = OBJ_ID_INITIAL;
+ gc_run_finalizers(objspace);
}
-VALUE
-rb_gc_impl_object_id(void *objspace_ptr, VALUE obj)
-{
- struct objspace *objspace = objspace_ptr;
-
- unsigned int lev = rb_gc_vm_lock();
-
- VALUE id;
- if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &id)) {
- RUBY_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
- }
- else {
- RUBY_ASSERT(!FL_TEST(obj, FL_SEEN_OBJ_ID));
-
- id = ULL2NUM(objspace->next_object_id);
- objspace->next_object_id += OBJ_ID_INCREMENT;
-
- st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
- st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
- FL_SET(obj, FL_SEEN_OBJ_ID);
- }
-
- rb_gc_vm_unlock(lev);
-
- return id;
-}
+// Forking
-VALUE
-rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id)
+void
+rb_gc_impl_before_fork(void *objspace_ptr)
{
struct objspace *objspace = objspace_ptr;
- VALUE obj;
- if (st_lookup(objspace->id_to_obj_tbl, object_id, &obj) &&
- !rb_gc_impl_garbage_object_p(objspace, obj)) {
- return obj;
+ retry:
+ objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+
+ /* At this point, we know that all the Ractors are paused because of the
+ * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point,
+ * one or more Ractors could be paused there. However, mmtk_before_fork is
+ * not compatible with that because it assumes that the MMTk workers are idle,
+ * but the workers are not idle because they are busy working on a GC.
+ *
+ * This essentially implements a trylock. It will optimistically lock but will
+ * release the lock if it detects that any other Ractors are waiting in
+ * rb_mmtk_block_for_gc.
+ */
+ rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count);
+ if (mutator_blocking_count != 0) {
+ RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
+ goto retry;
}
- if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(objspace->next_object_id))) {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
- else {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
-}
-
-// Forking
-
-void
-rb_gc_impl_before_fork(void *objspace_ptr)
-{
mmtk_before_fork();
}
void
rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
{
+ struct objspace *objspace = objspace_ptr;
+
mmtk_after_fork(rb_gc_get_ractor_newobj_cache());
+
+ RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
}
// Statistics
@@ -1235,6 +1489,7 @@ rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key)
enum gc_stat_sym {
gc_stat_sym_count,
+ gc_stat_sym_moving_gc_count,
gc_stat_sym_time,
gc_stat_sym_total_allocated_objects,
gc_stat_sym_total_bytes,
@@ -1242,6 +1497,7 @@ enum gc_stat_sym {
gc_stat_sym_free_bytes,
gc_stat_sym_starting_heap_address,
gc_stat_sym_last_heap_address,
+ gc_stat_sym_weak_references_count,
gc_stat_sym_last
};
@@ -1253,6 +1509,7 @@ setup_gc_stat_symbols(void)
if (gc_stat_symbols[0] == 0) {
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
S(count);
+ S(moving_gc_count);
S(time);
S(total_allocated_objects);
S(total_bytes);
@@ -1260,6 +1517,7 @@ setup_gc_stat_symbols(void)
S(free_bytes);
S(starting_heap_address);
S(last_heap_address);
+ S(weak_references_count);
}
}
@@ -1288,6 +1546,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
SET(count, objspace->gc_count);
+ SET(moving_gc_count, objspace->moving_gc_count);
SET(time, objspace->total_gc_time / (1000 * 1000));
SET(total_allocated_objects, objspace->total_allocated_objects);
SET(total_bytes, mmtk_total_bytes());
@@ -1295,6 +1554,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
SET(free_bytes, mmtk_free_bytes());
SET(starting_heap_address, (size_t)mmtk_starting_heap_address());
SET(last_heap_address, (size_t)mmtk_last_heap_address());
+ SET(weak_references_count, mmtk_weak_references_count());
#undef SET
if (!NIL_P(key)) {
@@ -1308,24 +1568,55 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
VALUE
rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
+ if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) {
+ int heap_idx = FIX2INT(heap_name);
+ if (heap_idx < 0 || heap_idx >= MMTK_HEAP_COUNT) {
+ rb_raise(rb_eArgError, "size pool index out of range");
+ }
+
+ if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) {
+ return SIZET2NUM(heap_sizes[heap_idx]);
+ }
+
+ return Qundef;
+ }
+
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
return hash_or_sym;
}
- else {
- return Qundef;
- }
+
+ return Qundef;
}
// Miscellaneous
-#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 0
+#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 1
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
struct rb_gc_object_metadata_entry *
rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
- object_metadata_entries[0].name = 0;
- object_metadata_entries[0].val = 0;
+ static ID ID_object_id;
+
+ if (!ID_object_id) {
+#define I(s) ID_##s = rb_intern(#s);
+ I(object_id);
+#undef I
+ }
+
+ size_t n = 0;
+
+#define SET_ENTRY(na, v) do { \
+ MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
+ object_metadata_entries[n].name = ID_##na; \
+ object_metadata_entries[n].val = v; \
+ n++; \
+} while (0)
+
+ if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
+
+ object_metadata_entries[n].name = 0;
+ object_metadata_entries[n].val = 0;
return object_metadata_entries;
}
diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h
index 614b832af5..b11e2873e3 100644
--- a/gc/mmtk/mmtk.h
+++ b/gc/mmtk/mmtk.h
@@ -20,6 +20,11 @@ typedef void *MMTk_ObjectReference;
typedef void *MMTk_NullableObjectReference;
typedef uint32_t MMTk_AllocationSemantics;
+typedef struct MMTk_BumpPointer {
+ uintptr_t cursor;
+ uintptr_t limit;
+} MMTk_BumpPointer;
+
#define MMTk_OBJREF_OFFSET 8
@@ -28,7 +33,6 @@ typedef uint32_t MMTk_AllocationSemantics;
#define MMTk_GC_THREAD_KIND_WORKER 1
typedef struct MMTk_RubyBindingOptions {
- bool ractor_check_mode;
size_t suffix_size;
} MMTk_RubyBindingOptions;
@@ -55,22 +59,26 @@ typedef struct MMTk_RubyUpcalls {
void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls);
bool (*is_mutator)(void);
void (*stop_the_world)(void);
- void (*resume_mutators)(void);
+ void (*resume_mutators)(bool gc_may_move);
void (*block_for_gc)(MMTk_VMMutatorThread tls);
+ void (*before_updating_jit_code)(void);
+ void (*after_updating_jit_code)(void);
size_t (*number_of_mutators)(void);
void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data);
void (*scan_gc_roots)(void);
void (*scan_objspace)(void);
- void (*scan_roots_in_mutator_thread)(MMTk_VMMutatorThread mutator_tls,
- MMTk_VMWorkerThread worker_tls);
- void (*scan_object_ruby_style)(MMTk_ObjectReference object);
+ void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to);
+ void (*update_object_references)(MMTk_ObjectReference object);
void (*call_gc_mark_children)(MMTk_ObjectReference object);
+ void (*handle_weak_references)(MMTk_ObjectReference object, bool moving);
void (*call_obj_free)(MMTk_ObjectReference object);
size_t (*vm_live_bytes)(void);
- void (*update_global_tables)(int tbl_idx);
+ void (*update_global_tables)(int tbl_idx, bool moving);
int (*global_tables_count)(void);
void (*update_finalizer_table)(void);
- void (*update_obj_id_tables)(void);
+ bool (*special_const_p)(MMTk_ObjectReference object);
+ void (*mutator_thread_panic_handler)(void);
+ void (*gc_thread_panic_handler)(void);
} MMTk_RubyUpcalls;
typedef struct MMTk_RawVecOfObjRef {
@@ -86,14 +94,15 @@ bool mmtk_is_reachable(MMTk_ObjectReference object);
MMTk_Builder *mmtk_builder_default(void);
void mmtk_init_binding(MMTk_Builder *builder,
- const struct MMTk_RubyBindingOptions *_binding_options,
- const struct MMTk_RubyUpcalls *upcalls,
- MMTk_ObjectReference weak_reference_dead_value);
+ const struct MMTk_RubyBindingOptions *binding_options,
+ const struct MMTk_RubyUpcalls *upcalls);
void mmtk_initialize_collection(MMTk_VMThread tls);
MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls);
+MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m);
+
void mmtk_destroy_mutator(MMTk_Mutator *mutator);
void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive);
@@ -113,11 +122,17 @@ void mmtk_post_alloc(MMTk_Mutator *mutator,
size_t bytes,
MMTk_AllocationSemantics semantics);
-void mmtk_add_obj_free_candidate(MMTk_ObjectReference object);
+void mmtk_add_obj_free_candidates(const MMTk_ObjectReference *objects,
+ size_t count,
+ bool can_parallel_free);
+
+void mmtk_declare_weak_references(MMTk_ObjectReference object);
+
+bool mmtk_weak_references_alive_p(MMTk_ObjectReference object);
-void mmtk_mark_weak(MMTk_ObjectReference *ptr);
+size_t mmtk_weak_references_count(void);
-void mmtk_remove_weak(const MMTk_ObjectReference *ptr);
+void mmtk_register_pinning_obj(MMTk_ObjectReference obj);
void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object);
diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs
index fc9c899743..30890e0853 100644
--- a/gc/mmtk/src/abi.rs
+++ b/gc/mmtk/src/abi.rs
@@ -1,8 +1,12 @@
use crate::api::RubyMutator;
+use crate::extra_assert;
use crate::Ruby;
use libc::c_int;
use mmtk::scheduler::GCWorker;
-use mmtk::util::{Address, ObjectReference, VMMutatorThread, VMWorkerThread};
+use mmtk::util::Address;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMMutatorThread;
+use mmtk::util::VMWorkerThread;
// For the C binding
pub const OBJREF_OFFSET: usize = 8;
@@ -10,16 +14,38 @@ pub const MIN_OBJ_ALIGN: usize = 8; // Even on 32-bit machine. A Ruby object is
pub const GC_THREAD_KIND_WORKER: libc::c_int = 1;
-const HAS_MOVED_GIVTBL: usize = 1 << 63;
const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF;
-// Should keep in sync with C code.
-const RUBY_FL_EXIVAR: usize = 1 << 10;
-
// An opaque type for the C counterpart.
#[allow(non_camel_case_types)]
pub struct st_table;
+#[repr(C)]
+pub struct HiddenHeader {
+ pub prefix: usize,
+}
+
+impl HiddenHeader {
+ #[inline(always)]
+ pub fn is_sane(&self) -> bool {
+ self.prefix & !HIDDEN_SIZE_MASK == 0
+ }
+
+ #[inline(always)]
+ fn assert_sane(&self) {
+ extra_assert!(
+ self.is_sane(),
+ "Hidden header is corrupted: {:x}",
+ self.prefix
+ );
+ }
+
+ pub fn payload_size(&self) -> usize {
+ self.assert_sane();
+ self.prefix & HIDDEN_SIZE_MASK
+ }
+}
+
/// Provide convenient methods for accessing Ruby objects.
/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods.
pub struct RubyObjectAccess {
@@ -47,32 +73,17 @@ impl RubyObjectAccess {
self.suffix_addr() + Self::suffix_size()
}
- fn hidden_field(&self) -> Address {
- self.obj_start()
- }
-
- fn load_hidden_field(&self) -> usize {
- unsafe { self.hidden_field().load::<usize>() }
+ fn hidden_header(&self) -> &'static HiddenHeader {
+ unsafe { self.obj_start().as_ref() }
}
- fn update_hidden_field<F>(&self, f: F)
- where
- F: FnOnce(usize) -> usize,
- {
- let old_value = self.load_hidden_field();
- let new_value = f(old_value);
- unsafe {
- self.hidden_field().store(new_value);
- }
+ #[allow(unused)] // Maybe we need to mutate the hidden header in the future.
+ fn hidden_header_mut(&self) -> &'static mut HiddenHeader {
+ unsafe { self.obj_start().as_mut_ref() }
}
pub fn payload_size(&self) -> usize {
- self.load_hidden_field() & HIDDEN_SIZE_MASK
- }
-
- pub fn set_payload_size(&self, size: usize) {
- debug_assert!((size & HIDDEN_SIZE_MASK) == size);
- self.update_hidden_field(|old| old & !HIDDEN_SIZE_MASK | size & HIDDEN_SIZE_MASK);
+ self.hidden_header().payload_size()
}
fn flags_field(&self) -> Address {
@@ -83,22 +94,6 @@ impl RubyObjectAccess {
unsafe { self.flags_field().load::<usize>() }
}
- pub fn has_exivar_flag(&self) -> bool {
- (self.load_flags() & RUBY_FL_EXIVAR) != 0
- }
-
- pub fn has_moved_givtbl(&self) -> bool {
- (self.load_hidden_field() & HAS_MOVED_GIVTBL) != 0
- }
-
- pub fn set_has_moved_givtbl(&self) {
- self.update_hidden_field(|old| old | HAS_MOVED_GIVTBL)
- }
-
- pub fn clear_has_moved_givtbl(&self) {
- self.update_hidden_field(|old| old & !HAS_MOVED_GIVTBL)
- }
-
pub fn prefix_size() -> usize {
// Currently, a hidden size field of word size is placed before each object.
OBJREF_OFFSET
@@ -232,7 +227,7 @@ impl GCThreadTLS {
/// Has undefined behavior if `ptr` is invalid.
pub unsafe fn check_cast(ptr: *mut GCThreadTLS) -> &'static mut GCThreadTLS {
assert!(!ptr.is_null());
- let result = &mut *ptr;
+ let result = unsafe { &mut *ptr };
debug_assert!({
let kind = result.kind;
kind == GC_THREAD_KIND_WORKER
@@ -247,7 +242,7 @@ impl GCThreadTLS {
/// Has undefined behavior if `ptr` is invalid.
pub unsafe fn from_vwt_check(vwt: VMWorkerThread) -> &'static mut GCThreadTLS {
let ptr = Self::from_vwt(vwt);
- Self::check_cast(ptr)
+ unsafe { Self::check_cast(ptr) }
}
#[allow(clippy::not_unsafe_ptr_arg_deref)] // `transmute` does not dereference pointer
@@ -283,7 +278,7 @@ impl RawVecOfObjRef {
///
/// This function turns raw pointer into a Vec without check.
pub unsafe fn into_vec(self) -> Vec<ObjectReference> {
- Vec::from_raw_parts(self.ptr, self.len, self.capa)
+ unsafe { Vec::from_raw_parts(self.ptr, self.len, self.capa) }
}
}
@@ -296,7 +291,6 @@ impl From<Vec<ObjectReference>> for RawVecOfObjRef {
#[repr(C)]
#[derive(Clone)]
pub struct RubyBindingOptions {
- pub ractor_check_mode: bool,
pub suffix_size: usize,
}
@@ -306,8 +300,10 @@ pub struct RubyUpcalls {
pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS),
pub is_mutator: extern "C" fn() -> bool,
pub stop_the_world: extern "C" fn(),
- pub resume_mutators: extern "C" fn(),
+ pub resume_mutators: extern "C" fn(gc_may_move: bool),
pub block_for_gc: extern "C" fn(tls: VMMutatorThread),
+ pub before_updating_jit_code: extern "C" fn(),
+ pub after_updating_jit_code: extern "C" fn(),
pub number_of_mutators: extern "C" fn() -> usize,
pub get_mutators: extern "C" fn(
visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void),
@@ -315,16 +311,18 @@ pub struct RubyUpcalls {
),
pub scan_gc_roots: extern "C" fn(),
pub scan_objspace: extern "C" fn(),
- pub scan_roots_in_mutator_thread:
- extern "C" fn(mutator_tls: VMMutatorThread, worker_tls: VMWorkerThread),
- pub scan_object_ruby_style: extern "C" fn(object: ObjectReference),
+ pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference),
+ pub update_object_references: extern "C" fn(object: ObjectReference),
pub call_gc_mark_children: extern "C" fn(object: ObjectReference),
+ pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool),
pub call_obj_free: extern "C" fn(object: ObjectReference),
pub vm_live_bytes: extern "C" fn() -> usize,
- pub update_global_tables: extern "C" fn(tbl_idx: c_int),
+ pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool),
pub global_tables_count: extern "C" fn() -> c_int,
pub update_finalizer_table: extern "C" fn(),
- pub update_obj_id_tables: extern "C" fn(),
+ pub special_const_p: extern "C" fn(object: ObjectReference) -> bool,
+ pub mutator_thread_panic_handler: extern "C" fn(),
+ pub gc_thread_panic_handler: extern "C" fn(),
}
unsafe impl Sync for RubyUpcalls {}
diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs
index c15996727e..c0540fe0c8 100644
--- a/gc/mmtk/src/api.rs
+++ b/gc/mmtk/src/api.rs
@@ -1,16 +1,28 @@
-use std::sync::atomic::Ordering;
+// Functions in this module are unsafe for one reason:
+// They are called by C functions and they need to pass raw pointers to Rust.
+#![allow(clippy::missing_safety_doc)]
+
+use mmtk::util::alloc::BumpPointer;
+use mmtk::util::alloc::ImmixAllocator;
+use mmtk::util::conversions;
use mmtk::util::options::PlanSelector;
+use std::str::FromStr;
+use std::sync::atomic::Ordering;
use crate::abi::RawVecOfObjRef;
use crate::abi::RubyBindingOptions;
use crate::abi::RubyUpcalls;
use crate::binding;
use crate::binding::RubyBinding;
+use crate::heap::CpuHeapTriggerConfig;
+use crate::heap::RubyHeapTriggerConfig;
+use crate::heap::CPU_HEAP_TRIGGER_CONFIG;
+use crate::heap::RUBY_HEAP_TRIGGER_CONFIG;
use crate::mmtk;
-use crate::Ruby;
-use crate::RubySlot;
use crate::utils::default_heap_max;
use crate::utils::parse_capacity;
+use crate::Ruby;
+use crate::RubySlot;
use mmtk::memory_manager;
use mmtk::memory_manager::mmtk_init;
use mmtk::util::constants::MIN_OBJECT_SIZE;
@@ -37,73 +49,139 @@ pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool {
// =============== Bootup ===============
-fn mmtk_builder_default_parse_threads() -> usize {
- let threads_str = std::env::var("MMTK_THREADS")
- .unwrap_or("0".to_string());
-
- threads_str
- .parse::<usize>()
- .unwrap_or_else(|_err| {
- eprintln!("[FATAL] Invalid MMTK_THREADS {}", threads_str);
+fn parse_env_var_with<T, F: FnOnce(&str) -> Option<T>>(key: &str, parse: F) -> Option<T> {
+ let val = match std::env::var(key) {
+ Ok(val) => val,
+ Err(std::env::VarError::NotPresent) => return None,
+ Err(std::env::VarError::NotUnicode(os_string)) => {
+ eprintln!("[FATAL] Invalid {key} {os_string:?}");
std::process::exit(1);
- })
-}
+ }
+ };
-fn mmtk_builder_default_parse_heap_min() -> usize {
- const DEFAULT_HEAP_MIN: usize = 1 << 20;
+ let parsed = parse(&val).unwrap_or_else(|| {
+ eprintln!("[FATAL] Invalid {key} {val}");
+ std::process::exit(1);
+ });
- let heap_min_str = std::env::var("MMTK_HEAP_MIN")
- .unwrap_or(DEFAULT_HEAP_MIN.to_string());
+ Some(parsed)
+}
- let size = parse_capacity(&heap_min_str, 0);
- if size == 0 {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MIN {}", heap_min_str);
- std::process::exit(1);
- }
+fn parse_env_var<T: FromStr>(key: &str) -> Option<T> {
+ parse_env_var_with(key, |s| s.parse().ok())
+}
- size
+fn mmtk_builder_default_parse_threads() -> Option<usize> {
+ parse_env_var("MMTK_THREADS")
+}
+
+fn mmtk_builder_default_parse_heap_min() -> usize {
+ const DEFAULT_HEAP_MIN: usize = 1 << 20;
+ parse_env_var_with("MMTK_HEAP_MIN", parse_capacity).unwrap_or(DEFAULT_HEAP_MIN)
}
fn mmtk_builder_default_parse_heap_max() -> usize {
- let heap_max_str = std::env::var("MMTK_HEAP_MAX")
- .unwrap_or(default_heap_max().to_string());
+ parse_env_var_with("MMTK_HEAP_MAX", parse_capacity).unwrap_or_else(default_heap_max)
+}
- let size = parse_capacity(&heap_max_str, 0);
- if size == 0 {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MAX {}", heap_max_str);
- std::process::exit(1);
- }
+fn parse_float_env_var(key: &str, default: f64, min: f64, max: f64) -> f64 {
+ parse_env_var_with(key, |s| {
+ let mut float = f64::from_str(s).unwrap_or(default);
+
+ if float <= min {
+ eprintln!(
+ "{key} has value {float} which must be greater than {min}, using default instead"
+ );
+ float = default;
+ }
- size
+ if float >= max {
+ eprintln!(
+ "{key} has value {float} which must be less than {max}, using default instead"
+ );
+ float = default;
+ }
+
+ Some(float)
+ })
+ .unwrap_or(default)
}
fn mmtk_builder_default_parse_heap_mode(heap_min: usize, heap_max: usize) -> GCTriggerSelector {
- let heap_mode_str = std::env::var("MMTK_HEAP_MODE")
- .unwrap_or("dynamic".to_string());
-
- match heap_mode_str.as_str() {
- "fixed" => GCTriggerSelector::FixedHeapSize(heap_max),
- "dynamic" => GCTriggerSelector::DynamicHeapSize(heap_min, heap_max),
- _ => {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MODE {}", heap_mode_str);
- std::process::exit(1);
+ let make_fixed = || GCTriggerSelector::FixedHeapSize(heap_max);
+ let make_dynamic = || GCTriggerSelector::DynamicHeapSize(heap_min, heap_max);
+
+ parse_env_var_with("MMTK_HEAP_MODE", |s| match s {
+ "fixed" => Some(make_fixed()),
+ "dynamic" => Some(make_dynamic()),
+ "ruby" => {
+ let min_ratio = parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", 0.2, 0.0, 1.0);
+ let goal_ratio =
+ parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", 0.4, min_ratio, 1.0);
+ let max_ratio =
+ parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", 0.65, goal_ratio, 1.0);
+
+ crate::heap::RUBY_HEAP_TRIGGER_CONFIG
+ .set(RubyHeapTriggerConfig {
+ min_heap_pages: conversions::bytes_to_pages_up(heap_min),
+ max_heap_pages: conversions::bytes_to_pages_up(heap_max),
+ heap_pages_min_ratio: min_ratio,
+ heap_pages_goal_ratio: goal_ratio,
+ heap_pages_max_ratio: max_ratio,
+ })
+ .unwrap_or_else(|_| panic!("RUBY_HEAP_TRIGGER_CONFIG is already set"));
+
+ Some(GCTriggerSelector::Delegated)
}
- }
+ "cpu" => {
+ // CPU-overhead-driven heap sizing based on Tavakolisomeh et al.,
+ // "Heap Size Adjustment with CPU Control", MPLR '23.
+ //
+ // Target is expressed as a percentage (0, 100) via
+ // `MMTK_GC_CPU_TARGET`. The paper recommends 15 for ZGC (a
+ // concurrent collector); we default to 5 for MMTk-Ruby. With
+ // MMTk's stop-the-world Immix, every percent of GC CPU is also
+ // a percent of wall-clock the mutator is blocked on, so a much
+ // smaller budget is appropriate. An empirical sweep across
+ // ruby-bench (railsbench, lobsters, psych-load, liquid-render,
+ // lee) found target=5 to be Pareto-optimal: ~6% geomean speedup
+ // vs. the `ruby` heap mode with effectively identical geomean
+ // peak RSS.
+ let target_percent = parse_float_env_var("MMTK_GC_CPU_TARGET", 5.0, 0.0, 100.0);
+ let window_size = parse_env_var::<usize>("MMTK_GC_CPU_WINDOW").unwrap_or(3);
+ let window_size = window_size.max(1);
+
+ let min_heap_pages = conversions::bytes_to_pages_up(heap_min);
+ let max_heap_pages = conversions::bytes_to_pages_up(heap_max);
+ // Start at the min heap size, as the other delegated triggers do.
+ // The control loop will adjust from here after the first GC cycle.
+ let initial_heap_pages = min_heap_pages;
+
+ CPU_HEAP_TRIGGER_CONFIG
+ .set(CpuHeapTriggerConfig {
+ min_heap_pages,
+ max_heap_pages,
+ initial_heap_pages,
+ target_gc_cpu: target_percent / 100.0,
+ window_size,
+ })
+ .unwrap_or_else(|_| panic!("CPU_HEAP_TRIGGER_CONFIG is already set"));
+
+ Some(GCTriggerSelector::Delegated)
+ }
+ _ => None,
+ })
+ .unwrap_or_else(make_dynamic)
}
fn mmtk_builder_default_parse_plan() -> PlanSelector {
- let plan_str = std::env::var("MMTK_PLAN")
- .unwrap_or("Immix".to_string());
-
- match plan_str.as_str() {
- "NoGC" => PlanSelector::NoGC,
- "MarkSweep" => PlanSelector::MarkSweep,
- "Immix" => PlanSelector::Immix,
- _ => {
- eprintln!("[FATAL] Invalid MMTK_PLAN {}", plan_str);
- std::process::exit(1);
- }
- }
+ parse_env_var_with("MMTK_PLAN", |s| match s {
+ "NoGC" => Some(PlanSelector::NoGC),
+ "MarkSweep" => Some(PlanSelector::MarkSweep),
+ "Immix" => Some(PlanSelector::Immix),
+ _ => None,
+ })
+ .unwrap_or(PlanSelector::Immix)
}
#[no_mangle]
@@ -111,9 +189,12 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
let mut builder = MMTKBuilder::new_no_env_vars();
builder.options.no_finalizer.set(true);
- let threads = mmtk_builder_default_parse_threads();
- if threads > 0 {
- builder.options.threads.set(threads);
+ if let Some(threads) = mmtk_builder_default_parse_threads() {
+ if !builder.options.threads.set(threads) {
+ // MMTk will validate it and reject 0.
+ eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}");
+ std::process::exit(1);
+ }
}
let heap_min = mmtk_builder_default_parse_heap_min();
@@ -121,11 +202,14 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
let heap_max = mmtk_builder_default_parse_heap_max();
if heap_min >= heap_max {
- eprintln!("[FATAL] MMTK_HEAP_MIN({}) >= MMTK_HEAP_MAX({})", heap_min, heap_max);
+ eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})");
std::process::exit(1);
}
- builder.options.gc_trigger.set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max));
+ builder
+ .options
+ .gc_trigger
+ .set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max));
builder.options.plan.set(mmtk_builder_default_parse_plan());
@@ -133,20 +217,26 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
}
#[no_mangle]
-pub extern "C" fn mmtk_init_binding(
+pub unsafe extern "C" fn mmtk_init_binding(
builder: *mut MMTKBuilder,
- _binding_options: *const RubyBindingOptions,
+ binding_options: *const RubyBindingOptions,
upcalls: *const RubyUpcalls,
- weak_reference_dead_value: ObjectReference,
) {
+ crate::MUTATOR_THREAD_PANIC_HANDLER
+ .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler)
+ .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized"));
+
crate::set_panic_hook();
- let builder = unsafe { Box::from_raw(builder) };
- let binding_options = RubyBindingOptions {ractor_check_mode: false, suffix_size: 0};
+ let builder: Box<MMTKBuilder> = unsafe { Box::from_raw(builder) };
+ let binding_options = unsafe { (*binding_options).clone() };
let mmtk_boxed = mmtk_init(&builder);
let mmtk_static = Box::leak(Box::new(mmtk_boxed));
- let binding = RubyBinding::new(mmtk_static, &binding_options, upcalls, weak_reference_dead_value);
+ let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls);
+ binding
+ .weak_proc
+ .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk));
crate::BINDING
.set(binding)
@@ -164,7 +254,25 @@ pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator {
}
#[no_mangle]
-pub extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) {
+pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer {
+ match *crate::BINDING.get().unwrap().mmtk.get_options().plan {
+ PlanSelector::Immix => {
+ let mutator: &mut Mutator<Ruby> = unsafe { &mut *m };
+ let allocator =
+ unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) };
+
+ if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() {
+ &mut immix_allocator.bump_pointer as *mut BumpPointer
+ } else {
+ panic!("Failed to get bump pointer allocator");
+ }
+ }
+ _ => std::ptr::null_mut(),
+ }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) {
// notify mmtk-core about destroyed mutator
memory_manager::destroy_mutator(unsafe { &mut *mutator });
// turn the ptr back to a box, and let Rust properly reclaim it
@@ -184,7 +292,9 @@ pub extern "C" fn mmtk_handle_user_collection_request(
#[no_mangle]
pub extern "C" fn mmtk_set_gc_enabled(enable: bool) {
- crate::CONFIGURATION.gc_enabled.store(enable, Ordering::Relaxed);
+ crate::CONFIGURATION
+ .gc_enabled
+ .store(enable, Ordering::Relaxed);
}
#[no_mangle]
@@ -195,7 +305,7 @@ pub extern "C" fn mmtk_gc_enabled_p() -> bool {
// =============== Object allocation ===============
#[no_mangle]
-pub extern "C" fn mmtk_alloc(
+pub unsafe extern "C" fn mmtk_alloc(
mutator: *mut RubyMutator,
size: usize,
align: usize,
@@ -213,7 +323,7 @@ pub extern "C" fn mmtk_alloc(
}
#[no_mangle]
-pub extern "C" fn mmtk_post_alloc(
+pub unsafe extern "C" fn mmtk_post_alloc(
mutator: *mut RubyMutator,
refer: ObjectReference,
bytes: usize,
@@ -222,28 +332,46 @@ pub extern "C" fn mmtk_post_alloc(
memory_manager::post_alloc::<Ruby>(unsafe { &mut *mutator }, refer, bytes, semantics)
}
-// TODO: Replace with buffered mmtk_add_obj_free_candidates
#[no_mangle]
-pub extern "C" fn mmtk_add_obj_free_candidate(object: ObjectReference) {
- binding().weak_proc.add_obj_free_candidate(object)
+pub unsafe extern "C" fn mmtk_add_obj_free_candidates(
+ objects: *const ObjectReference,
+ count: usize,
+ can_parallel_free: bool,
+) {
+ let objects = unsafe { std::slice::from_raw_parts(objects, count) };
+ binding()
+ .weak_proc
+ .add_obj_free_candidates_batch(objects, can_parallel_free)
+}
+
+// =============== Weak references ===============
+
+#[no_mangle]
+pub extern "C" fn mmtk_declare_weak_references(object: ObjectReference) {
+ binding().weak_proc.add_weak_reference(object);
}
-// =============== Marking ===============
+#[no_mangle]
+pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool {
+ object.is_reachable()
+}
#[no_mangle]
-pub extern "C" fn mmtk_mark_weak(ptr: &'static mut ObjectReference) {
- binding().weak_proc.add_weak_reference(ptr);
+pub extern "C" fn mmtk_weak_references_count() -> usize {
+ binding().weak_proc.weak_references_count()
}
+// =============== Compaction ===============
+
#[no_mangle]
-pub extern "C" fn mmtk_remove_weak(ptr: &ObjectReference) {
- binding().weak_proc.remove_weak_reference(ptr);
+pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) {
+ crate::binding().pinning_registry.register(obj);
}
// =============== Write barriers ===============
#[no_mangle]
-pub extern "C" fn mmtk_object_reference_write_post(
+pub unsafe extern "C" fn mmtk_object_reference_write_post(
mutator: *mut RubyMutator,
object: ObjectReference,
) {
@@ -347,7 +475,7 @@ pub extern "C" fn mmtk_plan() -> *const u8 {
PlanSelector::NoGC => NO_GC.as_ptr(),
PlanSelector::MarkSweep => MARK_SWEEP.as_ptr(),
PlanSelector::Immix => IMMIX.as_ptr(),
- _ => panic!("Unknown plan")
+ _ => panic!("Unknown plan"),
}
}
@@ -355,11 +483,21 @@ pub extern "C" fn mmtk_plan() -> *const u8 {
pub extern "C" fn mmtk_heap_mode() -> *const u8 {
static FIXED_HEAP: &[u8] = b"fixed\0";
static DYNAMIC_HEAP: &[u8] = b"dynamic\0";
+ static RUBY_HEAP: &[u8] = b"ruby\0";
+ static CPU_HEAP: &[u8] = b"cpu\0";
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(_) => FIXED_HEAP.as_ptr(),
GCTriggerSelector::DynamicHeapSize(_, _) => DYNAMIC_HEAP.as_ptr(),
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ // Two delegated triggers exist; disambiguate via the populated
+ // config singleton.
+ if CPU_HEAP_TRIGGER_CONFIG.get().is_some() {
+ CPU_HEAP.as_ptr()
+ } else {
+ RUBY_HEAP.as_ptr()
+ }
+ }
}
}
@@ -368,7 +506,18 @@ pub extern "C" fn mmtk_heap_min() -> usize {
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(_) => 0,
GCTriggerSelector::DynamicHeapSize(min_size, _) => min_size,
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() {
+ conversions::pages_to_bytes(cfg.min_heap_pages)
+ } else {
+ conversions::pages_to_bytes(
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("RUBY_HEAP_TRIGGER_CONFIG not set")
+ .min_heap_pages,
+ )
+ }
+ }
}
}
@@ -377,7 +526,18 @@ pub extern "C" fn mmtk_heap_max() -> usize {
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(max_size) => max_size,
GCTriggerSelector::DynamicHeapSize(_, max_size) => max_size,
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() {
+ conversions::pages_to_bytes(cfg.max_heap_pages)
+ } else {
+ conversions::pages_to_bytes(
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("RUBY_HEAP_TRIGGER_CONFIG not set")
+ .max_heap_pages,
+ )
+ }
+ }
}
}
diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs
index e0f8640e1c..36d4a992fd 100644
--- a/gc/mmtk/src/binding.rs
+++ b/gc/mmtk/src/binding.rs
@@ -9,6 +9,7 @@ use mmtk::MMTK;
use crate::abi;
use crate::abi::RubyBindingOptions;
+use crate::pinning_registry::PinningRegistry;
use crate::weak_proc::WeakProcessor;
use crate::Ruby;
@@ -54,10 +55,9 @@ pub struct RubyBinding {
pub upcalls: *const abi::RubyUpcalls,
pub plan_name: Mutex<Option<CString>>,
pub weak_proc: WeakProcessor,
+ pub pinning_registry: PinningRegistry,
pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>,
pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>,
-
- pub weak_reference_dead_value: ObjectReference,
}
unsafe impl Sync for RubyBinding {}
@@ -68,7 +68,6 @@ impl RubyBinding {
mmtk: &'static MMTK<Ruby>,
binding_options: &RubyBindingOptions,
upcalls: *const abi::RubyUpcalls,
- weak_reference_dead_value: ObjectReference,
) -> Self {
unsafe {
crate::BINDING_FAST.suffix_size = binding_options.suffix_size;
@@ -80,10 +79,9 @@ impl RubyBinding {
upcalls,
plan_name: Mutex::new(None),
weak_proc: WeakProcessor::new(),
+ pinning_registry: PinningRegistry::new(),
gc_thread_join_handles: Default::default(),
wb_unprotected_objects: Default::default(),
-
- weak_reference_dead_value
}
}
@@ -119,7 +117,7 @@ impl RubyBinding {
}
pub fn register_wb_unprotected_object(&self, object: ObjectReference) {
- debug!("Registering WB-unprotected object: {}", object);
+ debug!("Registering WB-unprotected object: {object}");
let mut objects = self.wb_unprotected_objects.lock().unwrap();
objects.insert(object);
}
diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs
index 0570b64e3a..648efa4e27 100644
--- a/gc/mmtk/src/collection.rs
+++ b/gc/mmtk/src/collection.rs
@@ -1,14 +1,26 @@
use crate::abi::GCThreadTLS;
use crate::api::RubyMutator;
-use crate::{mmtk, upcalls, Ruby};
+use crate::heap::CpuHeapTrigger;
+use crate::heap::RubyHeapTrigger;
+use crate::heap::CPU_HEAP_TRIGGER_CONFIG;
+use crate::mmtk;
+use crate::upcalls;
+use crate::Ruby;
use mmtk::memory_manager;
use mmtk::scheduler::*;
-use mmtk::util::{VMMutatorThread, VMThread, VMWorkerThread};
-use mmtk::vm::{Collection, GCThreadContext};
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::VMMutatorThread;
+use mmtk::util::VMThread;
+use mmtk::util::VMWorkerThread;
+use mmtk::vm::Collection;
+use mmtk::vm::GCThreadContext;
+use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::thread;
+static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false);
+
pub struct VMCollection {}
impl Collection<Ruby> for VMCollection {
@@ -16,11 +28,21 @@ impl Collection<Ruby> for VMCollection {
crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed)
}
- fn stop_all_mutators<F>(_tls: VMWorkerThread, mut mutator_visitor: F)
+ fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F)
where
F: FnMut(&'static mut mmtk::Mutator<Ruby>),
{
(upcalls().stop_the_world)();
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed);
+ (upcalls().before_updating_jit_code)();
+ } else {
+ CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed);
+ }
+
+ crate::binding().pinning_registry.pin_children(tls);
+
(upcalls().get_mutators)(
Self::notify_mutator_ready::<F>,
&mut mutator_visitor as *mut F as *mut _,
@@ -28,7 +50,13 @@ impl Collection<Ruby> for VMCollection {
}
fn resume_mutators(_tls: VMWorkerThread) {
- (upcalls().resume_mutators)();
+ let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed);
+
+ if current_gc_may_move {
+ (upcalls().after_updating_jit_code)();
+ }
+
+ (upcalls().resume_mutators)(current_gc_may_move);
}
fn block_for_gc(tls: VMMutatorThread) {
@@ -41,10 +69,7 @@ impl Collection<Ruby> for VMCollection {
.name("MMTk Worker Thread".to_string())
.spawn(move || {
let ordinal = worker.ordinal;
- debug!(
- "Hello! This is MMTk Worker Thread running! ordinal: {}",
- ordinal
- );
+ debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}");
crate::register_gc_thread(thread::current().id());
let ptr_worker = &mut *worker as *mut GCWorker<Ruby>;
let gc_thread_tls =
@@ -55,10 +80,7 @@ impl Collection<Ruby> for VMCollection {
GCThreadTLS::to_vwt(gc_thread_tls),
worker,
);
- debug!(
- "An MMTk Worker Thread is quitting. Good bye! ordinal: {}",
- ordinal
- );
+ debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}");
crate::unregister_gc_thread(thread::current().id());
})
.unwrap(),
@@ -73,6 +95,19 @@ impl Collection<Ruby> for VMCollection {
fn vm_live_bytes() -> usize {
(upcalls().vm_live_bytes)()
}
+
+ fn create_gc_trigger() -> Box<dyn GCTriggerPolicy<Ruby>> {
+ // `GCTriggerSelector::Delegated` is currently used by two different
+ // heap modes: `ruby` (the Ruby-like free-slot ratio trigger) and `cpu`
+ // (the CPU-overhead trigger from Tavakolisomeh et al., MPLR '23).
+ // Which one is active is determined by which `OnceCell` config the
+ // `MMTK_HEAP_MODE` parser populated.
+ if CPU_HEAP_TRIGGER_CONFIG.get().is_some() {
+ Box::new(CpuHeapTrigger::default())
+ } else {
+ Box::new(RubyHeapTrigger::default())
+ }
+ }
}
impl VMCollection {
diff --git a/gc/mmtk/src/heap/cpu_heap_trigger.rs b/gc/mmtk/src/heap/cpu_heap_trigger.rs
new file mode 100644
index 0000000000..ef5a79fe9a
--- /dev/null
+++ b/gc/mmtk/src/heap/cpu_heap_trigger.rs
@@ -0,0 +1,370 @@
+//! A GC trigger that adjusts the heap size based on the CPU overhead of GC.
+//!
+//! This is an implementation of the heap sizing policy described in
+//! Tavakolisomeh, Shimchenko, Österlund, Bruno, Ferreira, Wrigstad,
+//! "Heap Size Adjustment with CPU Control", MPLR '23.
+//! <https://doi.org/10.1145/3617651.3622988>
+//!
+//! The idea: rather than letting heap size control GC frequency, let a
+//! user-supplied *target GC CPU overhead* control the heap size. After each GC
+//! cycle, we measure the GC CPU overhead (fraction of process CPU time spent
+//! in GC) and compare it to the target. If GC is over budget we grow the heap
+//! (reducing GC frequency); if it is under budget we shrink the heap (trading
+//! memory for more frequent collections).
+//!
+//! ## Algorithm
+//!
+//! After each GC cycle we compute, using an average of the last `n` cycles:
+//!
+//! ```text
+//! GC_CPU = T_GC / T_APP (Eq. 1)
+//! overhead_error = GC_CPU - target (Eq. 2)
+//! sigmoid_error = 1 / (1 + e^(-overhead_error)) (Eq. 3)
+//! adjustment_factor = sigmoid_error + 0.5 (in (0.5, 1.5)) (Eq. 4)
+//! new_size = current_size * adjustment_factor (Eq. 5)
+//! ```
+//!
+//! where:
+//! - `T_GC` is the wall-clock duration of each GC cycle.
+//! - `T_APP` is process CPU time elapsed between consecutive GC cycles (sum of
+//! CPU time over all threads — mutators, GC workers, compilers, etc.), read
+//! via `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`.
+//!
+//! The final heap size is then clamped to the range
+//! `[max(1.1 * used, min_heap_pages), max_heap_pages]`, providing 10% headroom
+//! above current live memory to avoid triggering GC on an effectively-empty
+//! heap.
+//!
+//! ## Differences from the paper
+//!
+//! The paper targets ZGC, a concurrent generational collector. MMTk's Ruby
+//! binding currently ships stop-the-world collectors (Immix, MarkSweep). The
+//! paper's formula still applies: with a STW collector the process CPU time
+//! during GC closely tracks the wall-clock GC time, and mutator CPU time
+//! during the mutator phase is correctly attributed. For generational plans
+//! we skip nursery-only GCs, consistent with MemBalancer.
+
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Mutex;
+
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::heap::SpaceStats;
+use mmtk::Plan;
+use mmtk::MMTK;
+use once_cell::sync::OnceCell;
+
+use crate::Ruby;
+
+pub static CPU_HEAP_TRIGGER_CONFIG: OnceCell<CpuHeapTriggerConfig> = OnceCell::new();
+
+/// Configuration for the [`CpuHeapTrigger`].
+pub struct CpuHeapTriggerConfig {
+ /// Lower bound on heap size (in pages). The trigger will never shrink below
+ /// this value.
+ pub min_heap_pages: usize,
+ /// Upper bound on heap size (in pages). The trigger will never grow above
+ /// this value.
+ pub max_heap_pages: usize,
+ /// Initial heap size (in pages).
+ pub initial_heap_pages: usize,
+ /// Target GC CPU overhead as a fraction of total process CPU time. For
+ /// example, `0.15` means the policy will try to keep GC CPU usage near 15%.
+ /// Valid range: `(0.0, 1.0)`.
+ pub target_gc_cpu: f64,
+ /// Number of recent GC cycles averaged together when computing the CPU
+ /// overhead signal. Smoothes out short-term fluctuations. The paper uses 3.
+ pub window_size: usize,
+}
+
+/// A single GC cycle's timing measurements.
+#[derive(Clone, Copy, Debug, Default)]
+struct GcSample {
+ /// Wall-clock seconds spent inside this GC cycle.
+ gc_seconds: f64,
+ /// Seconds of process CPU time elapsed since the previous GC cycle ended.
+ /// This covers both mutator time and (on multi-threaded mutators) any
+ /// mutator CPU time consumed in parallel with the previous GC.
+ app_cpu_seconds: f64,
+}
+
+struct CpuHeapTriggerState {
+ /// Ring buffer of the last `window_size` samples. Oldest-first.
+ samples: Vec<GcSample>,
+ /// Wall-clock time when the current GC cycle started. `None` when no GC is
+ /// in progress.
+ gc_start_wall: Option<std::time::Instant>,
+ /// Process CPU time (seconds) recorded at the end of the previous GC
+ /// cycle. `None` until the first cycle completes.
+ last_gc_end_cpu: Option<f64>,
+}
+
+impl CpuHeapTriggerState {
+ fn new() -> Self {
+ Self {
+ samples: Vec::new(),
+ gc_start_wall: None,
+ last_gc_end_cpu: None,
+ }
+ }
+
+ /// Pushes a new sample, dropping the oldest when the window is full.
+ fn push_sample(&mut self, sample: GcSample, window_size: usize) {
+ if self.samples.len() >= window_size {
+ self.samples.remove(0);
+ }
+ self.samples.push(sample);
+ }
+
+ /// Returns the arithmetic mean GC CPU overhead across the window, or
+ /// `None` if we don't yet have a full sample (which happens on the first
+ /// GC cycle — we have no baseline for `app_cpu_seconds`).
+ fn mean_gc_cpu(&self) -> Option<f64> {
+ if self.samples.is_empty() {
+ return None;
+ }
+ let total_gc: f64 = self.samples.iter().map(|s| s.gc_seconds).sum();
+ let total_app: f64 = self.samples.iter().map(|s| s.app_cpu_seconds).sum();
+ if total_app <= 0.0 {
+ return None;
+ }
+ Some(total_gc / total_app)
+ }
+}
+
+pub struct CpuHeapTrigger {
+ /// Target heap size in pages. Updated at the end of each GC cycle.
+ target_heap_pages: AtomicUsize,
+ /// Mutable timing state. Wrapped in a `Mutex` because `on_gc_start` and
+ /// `on_gc_end` are the only mutation sites and they are not on an
+ /// allocation hot path; avoiding the complexity of lock-free state is
+ /// worth the trivial contention.
+ state: Mutex<CpuHeapTriggerState>,
+}
+
+impl Default for CpuHeapTrigger {
+ fn default() -> Self {
+ let cfg = Self::get_config();
+ Self {
+ target_heap_pages: AtomicUsize::new(cfg.initial_heap_pages),
+ state: Mutex::new(CpuHeapTriggerState::new()),
+ }
+ }
+}
+
+impl GCTriggerPolicy<Ruby> for CpuHeapTrigger {
+ fn is_gc_required(
+ &self,
+ space_full: bool,
+ space: Option<SpaceStats<Ruby>>,
+ plan: &dyn Plan<VM = Ruby>,
+ ) -> bool {
+ // Let the plan decide, matching the other triggers.
+ plan.collection_required(space_full, space)
+ }
+
+ fn on_gc_start(&self, _mmtk: &'static MMTK<Ruby>) {
+ let mut state = self.state.lock().unwrap();
+ state.gc_start_wall = Some(std::time::Instant::now());
+ }
+
+ fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) {
+ // Skip nursery-only GCs for generational plans. The heap resizing
+ // decision is driven by the (much more expensive) full collections
+ // where the signal-to-noise ratio is high enough to be useful.
+ if let Some(gen_plan) = mmtk.get_plan().generational() {
+ if gen_plan.is_current_gc_nursery() {
+ return;
+ }
+ }
+
+ let cfg = Self::get_config();
+ let gc_end_cpu = process_cpu_time_seconds();
+
+ let mut state = self.state.lock().unwrap();
+
+ // Duration of this GC cycle (wall clock).
+ let gc_seconds = state
+ .gc_start_wall
+ .take()
+ .map(|start| start.elapsed().as_secs_f64())
+ .unwrap_or(0.0);
+
+ // Process CPU time elapsed since the previous GC cycle ended. We
+ // require at least one previous end timestamp to produce a valid
+ // sample — without it we cannot compute `T_APP`.
+ if let (Some(last_end), Some(now)) = (state.last_gc_end_cpu, gc_end_cpu) {
+ let app_cpu_seconds = (now - last_end).max(0.0);
+ // Only record non-degenerate samples to avoid poisoning the window
+ // with zero-time entries from back-to-back GCs.
+ if app_cpu_seconds > 0.0 {
+ state.push_sample(
+ GcSample {
+ gc_seconds,
+ app_cpu_seconds,
+ },
+ cfg.window_size,
+ );
+ }
+ }
+ state.last_gc_end_cpu = gc_end_cpu;
+
+ // Compute the new heap size only when we have samples to average over.
+ if let Some(gc_cpu) = state.mean_gc_cpu() {
+ // Drop the lock before doing the (relatively cheap) math and
+ // atomic update; nothing below needs the state.
+ drop(state);
+
+ let overhead_error = gc_cpu - cfg.target_gc_cpu; // Eq. (2)
+ let sigmoid_error = sigmoid(overhead_error); // Eq. (3)
+ let adjustment_factor = sigmoid_error + 0.5; // Eq. (4), range (0.5, 1.5)
+
+ let current = self.target_heap_pages.load(Ordering::Relaxed);
+ let suggested = ((current as f64) * adjustment_factor) as usize; // Eq. (5)
+
+ // Clamp:
+ // - upper bound: configured max
+ // - lower bound: max(1.1 * used, min) — 10% headroom above current
+ // live memory, so we never request a heap so small that GC is
+ // triggered immediately on return from this one.
+ let used = mmtk.get_plan().get_used_pages();
+ let floor = ((used as f64) * 1.1).ceil() as usize;
+ let lower = floor.max(cfg.min_heap_pages).min(cfg.max_heap_pages);
+ let upper = cfg.max_heap_pages;
+ let new_target = suggested.clamp(lower, upper);
+
+ self.target_heap_pages.store(new_target, Ordering::Relaxed);
+
+ info!(
+ "CpuHeapTrigger: gc_cpu={:.4} target={:.4} factor={:.4} \
+ pages {} -> {} (used={}, clamp=[{}, {}])",
+ gc_cpu,
+ cfg.target_gc_cpu,
+ adjustment_factor,
+ current,
+ new_target,
+ used,
+ lower,
+ upper
+ );
+ }
+ }
+
+ fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool {
+ plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_current_heap_size_in_pages(&self) -> usize {
+ self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_max_heap_size_in_pages(&self) -> usize {
+ Self::get_config().max_heap_pages
+ }
+
+ fn can_heap_size_grow(&self) -> bool {
+ self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages
+ }
+}
+
+impl CpuHeapTrigger {
+ fn get_config<'b>() -> &'b CpuHeapTriggerConfig {
+ CPU_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("Attempt to use CPU_HEAP_TRIGGER_CONFIG before it is initialized")
+ }
+}
+
+/// Standard logistic sigmoid. Returns 0.5 when x == 0, asymptotes to 0 and 1.
+fn sigmoid(x: f64) -> f64 {
+ 1.0 / (1.0 + (-x).exp())
+}
+
+/// Reads the process-wide CPU time as a floating-point number of seconds,
+/// summed across all threads of this process. Returns `None` if the clock
+/// query fails (which should be essentially impossible on supported
+/// platforms).
+fn process_cpu_time_seconds() -> Option<f64> {
+ let mut ts = libc::timespec {
+ tv_sec: 0,
+ tv_nsec: 0,
+ };
+ // SAFETY: `clock_gettime` writes exactly `sizeof(timespec)` bytes to the
+ // pointer we pass, which is a valid local stack allocation.
+ let rc = unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) };
+ if rc != 0 {
+ return None;
+ }
+ Some((ts.tv_sec as f64) + (ts.tv_nsec as f64) / 1_000_000_000.0)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn sigmoid_is_well_behaved() {
+ assert!((sigmoid(0.0) - 0.5).abs() < 1e-12);
+ assert!(sigmoid(-100.0) < 1e-9);
+ assert!(sigmoid(100.0) > 1.0 - 1e-9);
+ // Monotonic.
+ assert!(sigmoid(-1.0) < sigmoid(0.0));
+ assert!(sigmoid(0.0) < sigmoid(1.0));
+ }
+
+ #[test]
+ fn adjustment_factor_is_within_paper_bounds() {
+ // Eq. (4): adjustment_factor = sigmoid(e) + 0.5 must lie in (0.5, 1.5).
+ for e in [-10.0_f64, -1.0, 0.0, 1.0, 10.0] {
+ let f = sigmoid(e) + 0.5;
+ assert!(f > 0.5 && f < 1.5, "factor {f} out of range for e={e}");
+ }
+ }
+
+ #[test]
+ fn mean_gc_cpu_is_total_weighted() {
+ let mut state = CpuHeapTriggerState::new();
+ state.push_sample(
+ GcSample {
+ gc_seconds: 1.0,
+ app_cpu_seconds: 10.0,
+ },
+ 3,
+ );
+ state.push_sample(
+ GcSample {
+ gc_seconds: 3.0,
+ app_cpu_seconds: 10.0,
+ },
+ 3,
+ );
+ // (1 + 3) / (10 + 10) = 0.2
+ assert!((state.mean_gc_cpu().unwrap() - 0.2).abs() < 1e-12);
+ }
+
+ #[test]
+ fn window_drops_oldest() {
+ let mut state = CpuHeapTriggerState::new();
+ for i in 0..5 {
+ state.push_sample(
+ GcSample {
+ gc_seconds: i as f64,
+ app_cpu_seconds: 1.0,
+ },
+ 3,
+ );
+ }
+ assert_eq!(state.samples.len(), 3);
+ // After pushing 0,1,2,3,4 with window 3, we should have [2,3,4].
+ assert_eq!(state.samples[0].gc_seconds, 2.0);
+ assert_eq!(state.samples[2].gc_seconds, 4.0);
+ }
+
+ #[test]
+ fn no_sample_without_prior_gc() {
+ // First GC cycle cannot produce a sample (no previous end time). The
+ // push happens only when last_gc_end_cpu is Some.
+ let state = CpuHeapTriggerState::new();
+ assert!(state.mean_gc_cpu().is_none());
+ }
+}
diff --git a/gc/mmtk/src/heap/mod.rs b/gc/mmtk/src/heap/mod.rs
new file mode 100644
index 0000000000..05a35efb23
--- /dev/null
+++ b/gc/mmtk/src/heap/mod.rs
@@ -0,0 +1,9 @@
+mod cpu_heap_trigger;
+mod ruby_heap_trigger;
+
+pub use cpu_heap_trigger::CpuHeapTrigger;
+pub use cpu_heap_trigger::CpuHeapTriggerConfig;
+pub use cpu_heap_trigger::CPU_HEAP_TRIGGER_CONFIG;
+pub use ruby_heap_trigger::RubyHeapTrigger;
+pub use ruby_heap_trigger::RubyHeapTriggerConfig;
+pub use ruby_heap_trigger::RUBY_HEAP_TRIGGER_CONFIG;
diff --git a/gc/mmtk/src/heap/ruby_heap_trigger.rs b/gc/mmtk/src/heap/ruby_heap_trigger.rs
new file mode 100644
index 0000000000..fe1130043d
--- /dev/null
+++ b/gc/mmtk/src/heap/ruby_heap_trigger.rs
@@ -0,0 +1,105 @@
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::heap::SpaceStats;
+use mmtk::Plan;
+use mmtk::MMTK;
+use once_cell::sync::OnceCell;
+
+use crate::Ruby;
+
+pub static RUBY_HEAP_TRIGGER_CONFIG: OnceCell<RubyHeapTriggerConfig> = OnceCell::new();
+
+pub struct RubyHeapTriggerConfig {
+ /// Min heap size
+ pub min_heap_pages: usize,
+ /// Max heap size
+ pub max_heap_pages: usize,
+ /// Minimum ratio of empty space after a GC before the heap will grow
+ pub heap_pages_min_ratio: f64,
+ /// Ratio the heap will grow by
+ pub heap_pages_goal_ratio: f64,
+ /// Maximum ratio of empty space after a GC before the heap will shrink
+ pub heap_pages_max_ratio: f64,
+}
+
+pub struct RubyHeapTrigger {
+ /// Target number of heap pages
+ target_heap_pages: AtomicUsize,
+}
+
+impl GCTriggerPolicy<Ruby> for RubyHeapTrigger {
+ fn is_gc_required(
+ &self,
+ space_full: bool,
+ space: Option<SpaceStats<Ruby>>,
+ plan: &dyn Plan<VM = Ruby>,
+ ) -> bool {
+ // Let the plan decide
+ plan.collection_required(space_full, space)
+ }
+
+ fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) {
+ if let Some(plan) = mmtk.get_plan().generational() {
+ if plan.is_current_gc_nursery() {
+ // Nursery GC
+ } else {
+ // Full GC
+ }
+
+ panic!("TODO: support for generational GC not implemented")
+ } else {
+ let used_pages = mmtk.get_plan().get_used_pages();
+
+ let target_min =
+ (used_pages as f64 * (1.0 + Self::get_config().heap_pages_min_ratio)) as usize;
+ let target_max =
+ (used_pages as f64 * (1.0 + Self::get_config().heap_pages_max_ratio)) as usize;
+ let new_target =
+ (((used_pages as f64) * (1.0 + Self::get_config().heap_pages_goal_ratio)) as usize)
+ .clamp(
+ Self::get_config().min_heap_pages,
+ Self::get_config().max_heap_pages,
+ );
+
+ if used_pages < target_min || used_pages > target_max {
+ self.target_heap_pages.store(new_target, Ordering::Relaxed);
+ }
+ }
+ }
+
+ fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool {
+ plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_current_heap_size_in_pages(&self) -> usize {
+ self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_max_heap_size_in_pages(&self) -> usize {
+ Self::get_config().max_heap_pages
+ }
+
+ fn can_heap_size_grow(&self) -> bool {
+ self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages
+ }
+}
+
+impl Default for RubyHeapTrigger {
+ fn default() -> Self {
+ let min_heap_pages = Self::get_config().min_heap_pages;
+
+ Self {
+ target_heap_pages: AtomicUsize::new(min_heap_pages),
+ }
+ }
+}
+
+impl RubyHeapTrigger {
+ fn get_config<'b>() -> &'b RubyHeapTriggerConfig {
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("Attempt to use RUBY_HEAP_TRIGGER_CONFIG before it is initialized")
+ }
+}
diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs
index 01497e9c42..52dc782051 100644
--- a/gc/mmtk/src/lib.rs
+++ b/gc/mmtk/src/lib.rs
@@ -1,3 +1,7 @@
+// Warn about unsafe operations in functions that are already marked as unsafe.
+// This will become default in Rust 2024 edition.
+#![warn(unsafe_op_in_unsafe_fn)]
+
extern crate libc;
extern crate mmtk;
#[macro_use]
@@ -10,8 +14,11 @@ use std::sync::Mutex;
use std::thread::ThreadId;
use abi::RubyUpcalls;
-use binding::{RubyBinding, RubyBindingFast, RubyConfiguration};
-use mmtk::vm::slot::{SimpleSlot, UnimplementedMemorySlice};
+use binding::RubyBinding;
+use binding::RubyBindingFast;
+use binding::RubyConfiguration;
+use mmtk::vm::slot::SimpleSlot;
+use mmtk::vm::slot::UnimplementedMemorySlice;
use mmtk::vm::VMBinding;
use mmtk::MMTK;
use once_cell::sync::OnceCell;
@@ -21,7 +28,9 @@ pub mod active_plan;
pub mod api;
pub mod binding;
pub mod collection;
+pub mod heap;
pub mod object_model;
+pub mod pinning_registry;
pub mod reference_glue;
pub mod scanning;
pub mod utils;
@@ -51,6 +60,11 @@ impl VMBinding for Ruby {
type VMMemorySlice = RubyMemorySlice;
}
+/// The callback for mutator thread panic handler (which calls rb_bug to output
+/// debugging information such as the Ruby backtrace and memory maps).
+/// This is set before BINDING is set because mmtk_init could panic.
+pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell<extern "C" fn()> = OnceCell::new();
+
/// The singleton object for the Ruby binding itself.
pub static BINDING: OnceCell<RubyBinding> = OnceCell::new();
@@ -112,8 +126,6 @@ fn handle_gc_thread_panic(panic_info: &PanicHookInfo) {
eprintln!("Unknown backtrace status: {s:?}");
}
}
-
- std::process::abort();
}
pub(crate) fn set_panic_hook() {
@@ -126,8 +138,24 @@ pub(crate) fn set_panic_hook() {
std::panic::set_hook(Box::new(move |panic_info| {
if is_gc_thread(std::thread::current().id()) {
handle_gc_thread_panic(panic_info);
+
+ (crate::binding().upcalls().gc_thread_panic_handler)();
} else {
old_hook(panic_info);
+ (crate::MUTATOR_THREAD_PANIC_HANDLER
+ .get()
+ .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))();
}
}));
}
+
+/// This kind of assertion is enabled if either building in debug mode or the
+/// "extra_assert" feature is enabled.
+#[macro_export]
+macro_rules! extra_assert {
+ ($($arg:tt)*) => {
+ if std::cfg!(any(debug_assertions, feature = "extra_assert")) {
+ std::assert!($($arg)*);
+ }
+ };
+}
diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs
index abeef1f2b9..d673ca11a0 100644
--- a/gc/mmtk/src/object_model.rs
+++ b/gc/mmtk/src/object_model.rs
@@ -1,8 +1,15 @@
-use crate::abi::{RubyObjectAccess, OBJREF_OFFSET};
-use crate::{abi, Ruby};
+use std::ptr::copy_nonoverlapping;
+
+use crate::abi;
+use crate::abi::RubyObjectAccess;
+use crate::abi::MIN_OBJ_ALIGN;
+use crate::abi::OBJREF_OFFSET;
+use crate::Ruby;
use mmtk::util::constants::BITS_IN_BYTE;
-use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext};
-use mmtk::util::{Address, ObjectReference};
+use mmtk::util::copy::CopySemantics;
+use mmtk::util::copy::GCWorkerCopyContext;
+use mmtk::util::Address;
+use mmtk::util::ObjectReference;
use mmtk::vm::*;
pub struct VMObjectModel {}
@@ -36,13 +43,39 @@ impl ObjectModel<Ruby> for VMObjectModel {
const NEED_VO_BITS_DURING_TRACING: bool = true;
fn copy(
- _from: ObjectReference,
- _semantics: CopySemantics,
- _copy_context: &mut GCWorkerCopyContext<Ruby>,
+ from: ObjectReference,
+ semantics: CopySemantics,
+ copy_context: &mut GCWorkerCopyContext<Ruby>,
) -> ObjectReference {
- unimplemented!(
- "Copying GC not currently supported"
- )
+ let from_acc = RubyObjectAccess::from_objref(from);
+ let from_start = from_acc.obj_start();
+ let object_size = from_acc.object_size();
+ let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics);
+ debug_assert!(!to_start.is_zero());
+ let to_payload = to_start.add(OBJREF_OFFSET);
+ unsafe {
+ copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size);
+ }
+ let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) };
+ copy_context.post_copy(to_obj, object_size, semantics);
+ trace!("Copied object from {} to {}", from, to_obj);
+
+ (crate::binding().upcalls().move_obj_during_marking)(from, to_obj);
+
+ #[cfg(feature = "clear_old_copy")]
+ {
+ trace!(
+ "Clearing old copy {} ({}-{})",
+ from,
+ from_start,
+ from_start + object_size
+ );
+ // For debug purpose, we clear the old copy so that if the Ruby VM reads from the old
+ // copy again, it will likely result in an error.
+ unsafe { std::ptr::write_bytes::<u8>(from_start.to_mut_ptr(), 0, object_size) }
+ }
+
+ to_obj
}
fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address {
diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs
new file mode 100644
index 0000000000..b498b508f1
--- /dev/null
+++ b/gc/mmtk/src/pinning_registry.rs
@@ -0,0 +1,187 @@
+use std::sync::Mutex;
+
+use mmtk::memory_manager;
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMWorkerThread;
+use mmtk::MMTK;
+
+use crate::abi::GCThreadTLS;
+use crate::upcalls;
+use crate::Ruby;
+
+pub struct PinningRegistry {
+ pinning_objs: Mutex<Vec<ObjectReference>>,
+ pinned_objs: Mutex<Vec<ObjectReference>>,
+}
+
+impl PinningRegistry {
+ pub fn new() -> Self {
+ Self {
+ pinning_objs: Default::default(),
+ pinned_objs: Default::default(),
+ }
+ }
+
+ pub fn register(&self, object: ObjectReference) {
+ let mut pinning_objs = self.pinning_objs.lock().unwrap();
+ pinning_objs.push(object);
+ }
+
+ pub fn pin_children(&self, tls: VMWorkerThread) {
+ if !crate::mmtk().get_plan().current_gc_may_move_object() {
+ log::debug!("The current GC is non-moving, skipping pinning children.");
+ return;
+ }
+
+ let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) };
+ let worker = gc_tls.worker();
+
+ let pinning_objs = self
+ .pinning_objs
+ .try_lock()
+ .expect("PinningRegistry should not have races during GC.");
+
+ let packet_size = 512;
+ let work_packets = pinning_objs
+ .chunks(packet_size)
+ .map(|chunk| {
+ Box::new(PinPinningChildren {
+ pinning_objs: chunk.to_vec(),
+ }) as _
+ })
+ .collect();
+
+ worker.scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(work_packets);
+ }
+
+ pub fn cleanup(&self, worker: &mut GCWorker<Ruby>) {
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(RemoveDeadPinnings);
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ let packet = {
+ let mut pinned_objs = self
+ .pinned_objs
+ .try_lock()
+ .expect("Unexpected contention on pinned_objs");
+ UnpinPinnedObjects {
+ objs: std::mem::take(&mut pinned_objs),
+ }
+ };
+
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(packet);
+ } else {
+ debug!("The current GC is non-moving, skipping unpinning objects.");
+ debug_assert_eq!(
+ {
+ let pinned_objs = self
+ .pinned_objs
+ .try_lock()
+ .expect("Unexpected contention on pinned_objs");
+ pinned_objs.len()
+ },
+ 0
+ );
+ }
+ }
+}
+
+impl Default for PinningRegistry {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+struct PinPinningChildren {
+ pinning_objs: Vec<ObjectReference>,
+}
+
+impl GCWork<Ruby> for PinPinningChildren {
+ fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) };
+ let mut pinned_objs = vec![];
+ let mut newly_pinned_objs = vec![];
+
+ let visit_object = |_worker, target_object: ObjectReference, pin| {
+ log::trace!(
+ " -> {} {}",
+ if pin { "(pin)" } else { " " },
+ target_object
+ );
+ if pin {
+ debug_assert!(
+ target_object.get_forwarded_object().is_none(),
+ "Trying to pin {target_object} but has been moved"
+ );
+
+ pinned_objs.push(target_object);
+ }
+ target_object
+ };
+
+ gc_tls
+ .object_closure
+ .set_temporarily_and_run_code(visit_object, || {
+ for obj in self.pinning_objs.iter().cloned() {
+ log::trace!(" Pinning: {}", obj);
+ (upcalls().call_gc_mark_children)(obj);
+ }
+ });
+
+ for target_object in pinned_objs {
+ if memory_manager::pin_object(target_object) {
+ newly_pinned_objs.push(target_object);
+ }
+ }
+
+ let mut pinned_objs = crate::binding()
+ .pinning_registry
+ .pinned_objs
+ .lock()
+ .unwrap();
+ pinned_objs.append(&mut newly_pinned_objs);
+ }
+}
+
+struct RemoveDeadPinnings;
+
+impl GCWork<Ruby> for RemoveDeadPinnings {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ log::debug!("Removing dead Pinnings...");
+
+ let registry = &crate::binding().pinning_registry;
+ {
+ let mut pinning_objs = registry
+ .pinning_objs
+ .try_lock()
+ .expect("PinningRegistry should not have races during GC.");
+
+ pinning_objs.retain_mut(|obj| {
+ if obj.is_live() {
+ let new_obj = obj.get_forwarded_object().unwrap_or(*obj);
+ *obj = new_obj;
+ true
+ } else {
+ log::trace!(" Dead Pinning removed: {}", *obj);
+ false
+ }
+ });
+ }
+ }
+}
+
+struct UnpinPinnedObjects {
+ objs: Vec<ObjectReference>,
+}
+
+impl GCWork<Ruby> for UnpinPinnedObjects {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ log::debug!("Unpinning pinned objects...");
+
+ for obj in self.objs.iter() {
+ let unpinned = memory_manager::unpin_object(*obj);
+ debug_assert!(unpinned);
+ }
+ }
+}
diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs
index 33466b9db6..355a2e7759 100644
--- a/gc/mmtk/src/scanning.rs
+++ b/gc/mmtk/src/scanning.rs
@@ -1,11 +1,20 @@
use crate::abi::GCThreadTLS;
+use crate::upcalls;
use crate::utils::ChunkedVecCollector;
-use crate::{upcalls, Ruby, RubySlot};
-use mmtk::scheduler::{GCWork, GCWorker, WorkBucketStage};
-use mmtk::util::{ObjectReference, VMWorkerThread};
-use mmtk::vm::{ObjectTracer, RootsWorkFactory, Scanning, SlotVisitor};
-use mmtk::{Mutator, MutatorContext};
+use crate::Ruby;
+use crate::RubySlot;
+use mmtk::memory_manager;
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMWorkerThread;
+use mmtk::vm::ObjectTracer;
+use mmtk::vm::RootsWorkFactory;
+use mmtk::vm::Scanning;
+use mmtk::vm::SlotVisitor;
+use mmtk::Mutator;
pub struct VMScanning {}
@@ -45,20 +54,33 @@ impl Scanning<Ruby> for VMScanning {
mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(),
"Destination is not an MMTk object. Src: {object} dst: {target_object}"
);
+
+ debug_assert!(
+ // If we are in a moving GC, all objects should be pinned by PinningRegistry.
+ // If it is requested that target_object be pinned but it is not pinned, then
+ // it is a bug because it could be moved.
+ if crate::mmtk().get_plan().current_gc_may_move_object() && pin {
+ memory_manager::is_pinned(target_object)
+ } else {
+ true
+ },
+ "Object {object} is trying to pin {target_object}"
+ );
+
let forwarded_target = object_tracer.trace_object(target_object);
if forwarded_target != target_object {
- trace!(
- " Forwarded target {} -> {}",
- target_object,
- forwarded_target
- );
+ trace!(" Forwarded target {target_object} -> {forwarded_target}");
}
forwarded_target
};
gc_tls
.object_closure
.set_temporarily_and_run_code(visit_object, || {
- (upcalls().scan_object_ruby_style)(object);
+ (upcalls().call_gc_mark_children)(object);
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ (upcalls().update_object_references)(object);
+ }
});
}
@@ -67,14 +89,13 @@ impl Scanning<Ruby> for VMScanning {
}
fn scan_roots_in_mutator_thread(
- tls: VMWorkerThread,
- mutator: &'static mut Mutator<Ruby>,
- mut factory: impl RootsWorkFactory<RubySlot>,
+ _tls: VMWorkerThread,
+ _mutator: &'static mut Mutator<Ruby>,
+ mut _factory: impl RootsWorkFactory<RubySlot>,
) {
- let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) };
- Self::collect_object_roots_in("scan_thread_root", gc_tls, &mut factory, || {
- (upcalls().scan_roots_in_mutator_thread)(mutator.get_tls(), tls);
- });
+ // Do nothing. All stacks (including Ruby stacks and machine stacks) are reachable from
+ // `rb_vm_t` -> ractor -> thread -> fiber -> stacks. It is part of `ScanGCRoots` which
+ // calls `rb_gc_mark_roots` -> `rb_vm_mark`.
}
fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory<RubySlot>) {
@@ -136,6 +157,7 @@ impl Scanning<Ruby> for VMScanning {
crate::binding()
.weak_proc
.process_weak_stuff(worker, tracer_context);
+ crate::binding().pinning_registry.cleanup(worker);
false
}
@@ -252,15 +274,15 @@ impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> {
VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || {
for object in self.objects.iter().copied() {
if object.is_reachable() {
- debug!(
- "[wb_unprot_roots] Visiting WB-unprotected object (parent): {}",
- object
- );
- (upcalls().scan_object_ruby_style)(object);
+ debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}");
+ (upcalls().call_gc_mark_children)(object);
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ (upcalls().update_object_references)(object);
+ }
} else {
debug!(
- "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {}",
- object
+ "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}"
);
}
}
diff --git a/gc/mmtk/src/utils.rs b/gc/mmtk/src/utils.rs
index de929c3952..d1979eaf58 100644
--- a/gc/mmtk/src/utils.rs
+++ b/gc/mmtk/src/utils.rs
@@ -1,10 +1,13 @@
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
use atomic_refcell::AtomicRefCell;
-use mmtk::scheduler::{GCWork, GCWorker, WorkBucketStage};
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
-use sysinfo::System;
use crate::Ruby;
+use sysinfo::System;
pub struct ChunkedVecCollector<T> {
vecs: Vec<Vec<T>>,
@@ -97,32 +100,29 @@ pub fn default_heap_max() -> usize {
.expect("Invalid Memory size") as usize
}
-pub fn parse_capacity(input: &String, default: usize) -> usize {
+pub fn parse_capacity(input: &str) -> Option<usize> {
let trimmed = input.trim();
const KIBIBYTE: usize = 1024;
const MEBIBYTE: usize = 1024 * KIBIBYTE;
const GIBIBYTE: usize = 1024 * MEBIBYTE;
- let (val, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) {
- (&trimmed[..pos], &trimmed[pos..])
+ let (number, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) {
+ trimmed.split_at(pos)
} else {
(trimmed, "")
};
- // 1MiB is the default heap size
- match (val, suffix) {
- (number, "GiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * GIBIBYTE))
- .unwrap_or(default),
- (number, "MiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * MEBIBYTE))
- .unwrap_or(default),
- (number, "KiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * KIBIBYTE))
- .unwrap_or(default),
- (number, suffix) if suffix.is_empty() => number.parse::<usize>().unwrap_or(default),
- (_, _) => default
+ let Ok(v) = number.parse::<usize>() else {
+ return None;
+ };
+
+ match suffix {
+ "GiB" => Some(v * GIBIBYTE),
+ "MiB" => Some(v * MEBIBYTE),
+ "KiB" => Some(v * KIBIBYTE),
+ "" => Some(v),
+ _ => None,
}
}
@@ -132,32 +132,30 @@ mod tests {
#[test]
fn test_parse_capacity_parses_bare_bytes() {
- assert_eq!(1234, parse_capacity(&String::from("1234"), 0));
+ assert_eq!(Some(1234), parse_capacity("1234"));
}
#[test]
fn test_parse_capacity_parses_kibibytes() {
- assert_eq!(10240, parse_capacity(&String::from("10KiB"), 0))
+ assert_eq!(Some(10240), parse_capacity("10KiB"));
}
#[test]
fn test_parse_capacity_parses_mebibytes() {
- assert_eq!(10485760, parse_capacity(&String::from("10MiB"), 0))
+ assert_eq!(Some(10485760), parse_capacity("10MiB"))
}
#[test]
fn test_parse_capacity_parses_gibibytes() {
- assert_eq!(10737418240, parse_capacity(&String::from("10GiB"), 0))
+ assert_eq!(Some(10737418240), parse_capacity("10GiB"))
}
#[test]
- fn test_parses_nonsense_value_as_default_max() {
- let default = 100;
-
- assert_eq!(default, parse_capacity(&String::from("notanumber"), default));
- assert_eq!(default, parse_capacity(&String::from("5tartswithanumber"), default));
- assert_eq!(default, parse_capacity(&String::from("number1nthemiddle"), default));
- assert_eq!(default, parse_capacity(&String::from("numberattheend111"), default));
- assert_eq!(default, parse_capacity(&String::from("mult1pl3numb3r5"), default));
+ fn test_parse_capacity_parses_nonsense_values() {
+ assert_eq!(None, parse_capacity("notanumber"));
+ assert_eq!(None, parse_capacity("5tartswithanumber"));
+ assert_eq!(None, parse_capacity("number1nthemiddle"));
+ assert_eq!(None, parse_capacity("numberattheend111"));
+ assert_eq!(None, parse_capacity("mult1pl3numb3r5"));
}
}
diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs
index 11f7f5abbf..d38dbe04a4 100644
--- a/gc/mmtk/src/weak_proc.rs
+++ b/gc/mmtk/src/weak_proc.rs
@@ -1,23 +1,23 @@
use std::sync::Mutex;
-use mmtk::{
- scheduler::{GCWork, GCWorker, WorkBucketStage},
- util::ObjectReference,
- vm::ObjectTracerContext,
-};
-
-use crate::{
- abi::GCThreadTLS,
- upcalls,
- Ruby,
-};
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::vm::ObjectTracerContext;
+
+use crate::abi::GCThreadTLS;
+use crate::upcalls;
+use crate::Ruby;
pub struct WeakProcessor {
+ non_parallel_obj_free_candidates: Mutex<Vec<ObjectReference>>,
+ parallel_obj_free_candidates: Vec<Mutex<Vec<ObjectReference>>>,
+
/// Objects that needs `obj_free` called when dying.
/// If it is a bottleneck, replace it with a lock-free data structure,
/// or add candidates in batch.
- obj_free_candidates: Mutex<Vec<ObjectReference>>,
- weak_references: Mutex<Vec<&'static mut ObjectReference>>,
+ weak_references: Mutex<Vec<ObjectReference>>,
}
impl Default for WeakProcessor {
@@ -29,47 +29,74 @@ impl Default for WeakProcessor {
impl WeakProcessor {
pub fn new() -> Self {
Self {
- obj_free_candidates: Mutex::new(Vec::new()),
+ non_parallel_obj_free_candidates: Mutex::new(Vec::new()),
+ parallel_obj_free_candidates: vec![Mutex::new(Vec::new())],
weak_references: Mutex::new(Vec::new()),
}
}
- /// Add an object as a candidate for `obj_free`.
- ///
- /// Multiple mutators can call it concurrently, so it has `&self`.
- pub fn add_obj_free_candidate(&self, object: ObjectReference) {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- obj_free_candidates.push(object);
+ pub fn init_parallel_obj_free_candidates(&mut self, num_workers: usize) {
+ debug_assert_eq!(self.parallel_obj_free_candidates.len(), 1);
+
+ for _ in 1..num_workers {
+ self.parallel_obj_free_candidates
+ .push(Mutex::new(Vec::new()));
+ }
}
- /// Add many objects as candidates for `obj_free`.
+ /// Add a batch of objects as candidates for `obj_free`.
///
- /// Multiple mutators can call it concurrently, so it has `&self`.
- pub fn add_obj_free_candidates(&self, objects: &[ObjectReference]) {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- for object in objects.iter().copied() {
- obj_free_candidates.push(object);
+ /// Amortizes mutex acquisition over the entire batch. Called when a
+ /// mutator's local buffer is flushed (buffer full or stop-the-world).
+ pub fn add_obj_free_candidates_batch(
+ &self,
+ objects: &[ObjectReference],
+ can_parallel_free: bool,
+ ) {
+ if objects.is_empty() {
+ return;
+ }
+
+ if can_parallel_free {
+ let num_buckets = self.parallel_obj_free_candidates.len();
+ for idx in 0..num_buckets {
+ let mut bucket = self.parallel_obj_free_candidates[idx].lock().unwrap();
+ for (i, &obj) in objects.iter().enumerate() {
+ if i % num_buckets == idx {
+ bucket.push(obj);
+ }
+ }
+ }
+ } else {
+ self.non_parallel_obj_free_candidates
+ .lock()
+ .unwrap()
+ .extend_from_slice(objects);
}
}
pub fn get_all_obj_free_candidates(&self) -> Vec<ObjectReference> {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- std::mem::take(obj_free_candidates.as_mut())
+ // let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
+ let mut all_obj_free_candidates = self
+ .non_parallel_obj_free_candidates
+ .lock()
+ .unwrap()
+ .to_vec();
+
+ for candidates_mutex in &self.parallel_obj_free_candidates {
+ all_obj_free_candidates.extend(candidates_mutex.lock().unwrap().to_vec());
+ }
+
+ std::mem::take(all_obj_free_candidates.as_mut())
}
- pub fn add_weak_reference(&self, ptr: &'static mut ObjectReference) {
+ pub fn add_weak_reference(&self, object: ObjectReference) {
let mut weak_references = self.weak_references.lock().unwrap();
- weak_references.push(ptr);
+ weak_references.push(object);
}
- pub fn remove_weak_reference(&self, ptr: &ObjectReference) {
- let mut weak_references = self.weak_references.lock().unwrap();
- for (i, curr_ptr) in weak_references.iter().enumerate() {
- if *curr_ptr == ptr {
- weak_references.swap_remove(i);
- break;
- }
- }
+ pub fn weak_references_count(&self) -> usize {
+ self.weak_references.lock().unwrap().len()
}
pub fn process_weak_stuff(
@@ -77,80 +104,133 @@ impl WeakProcessor {
worker: &mut GCWorker<Ruby>,
_tracer_context: impl ObjectTracerContext<Ruby>,
) {
- worker.add_work(WorkBucketStage::VMRefClosure, ProcessObjFreeCandidates);
+ worker.add_work(
+ WorkBucketStage::VMRefClosure,
+ ProcessNonParallelObjFreeCanadidates {},
+ );
+
+ for index in 0..self.parallel_obj_free_candidates.len() {
+ worker.add_work(
+ WorkBucketStage::VMRefClosure,
+ ProcessParallelObjFreeCandidates { index },
+ );
+ }
+
worker.add_work(WorkBucketStage::VMRefClosure, ProcessWeakReferences);
worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerObjIdTables);
let global_tables_count = (crate::upcalls().global_tables_count)();
let work_packets = (0..global_tables_count)
- .map(|i| {
- Box::new(UpdateGlobalTables { idx: i }) as _
- })
- .collect();
+ .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _)
+ .collect();
worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(work_packets);
- worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(vec![
- Box::new(UpdateWbUnprotectedObjectsList) as _,
- ]);
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure]
+ .bulk_add(vec![Box::new(UpdateWbUnprotectedObjectsList) as _]);
+ }
+}
+
+fn process_obj_free_candidates(obj_free_candidates: &mut Vec<ObjectReference>) {
+ // Process obj_free
+ let mut new_candidates = Vec::new();
+
+ for object in obj_free_candidates.iter().copied() {
+ if object.is_reachable() {
+ // Forward and add back to the candidate list.
+ let new_object = object.forward();
+ trace!("Forwarding obj_free candidate: {object} -> {new_object}");
+ new_candidates.push(new_object);
+ } else {
+ (upcalls().call_obj_free)(object);
+ }
+ }
+
+ *obj_free_candidates = new_candidates;
+}
+
+struct ProcessParallelObjFreeCandidates {
+ index: usize,
+}
+
+impl GCWork<Ruby> for ProcessParallelObjFreeCandidates {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+ let mut obj_free_candidates = crate::binding().weak_proc.parallel_obj_free_candidates
+ [self.index]
+ .try_lock()
+ .expect("Lock for parallel_obj_free_candidates should not be held");
+
+ process_obj_free_candidates(&mut obj_free_candidates);
}
}
-struct ProcessObjFreeCandidates;
+struct ProcessNonParallelObjFreeCanadidates;
-impl GCWork<Ruby> for ProcessObjFreeCandidates {
+impl GCWork<Ruby> for ProcessNonParallelObjFreeCanadidates {
fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
- // If it blocks, it is a bug.
let mut obj_free_candidates = crate::binding()
.weak_proc
- .obj_free_candidates
+ .non_parallel_obj_free_candidates
.try_lock()
- .expect("It's GC time. No mutators should hold this lock at this time.");
+ .expect("Lock for non_parallel_obj_free_candidates should not be held");
- let n_cands = obj_free_candidates.len();
+ process_obj_free_candidates(&mut obj_free_candidates);
+ }
+}
- debug!("Total: {} candidates", n_cands);
+struct ProcessWeakReferences;
- // Process obj_free
- let mut new_candidates = Vec::new();
+impl GCWork<Ruby> for ProcessWeakReferences {
+ fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ let gc_tls: &mut GCThreadTLS = unsafe { GCThreadTLS::from_vwt_check(worker.tls) };
- for object in obj_free_candidates.iter().copied() {
- if object.is_reachable() {
- // Forward and add back to the candidate list.
- let new_object = object.forward();
- trace!(
- "Forwarding obj_free candidate: {} -> {}",
- object,
- new_object
+ let visit_object = |_worker, target_object: ObjectReference, _pin| {
+ debug_assert!(
+ mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(),
+ "Destination is not an MMTk object"
);
- new_candidates.push(new_object);
- } else {
- (upcalls().call_obj_free)(object);
- }
- }
- *obj_free_candidates = new_candidates;
+ target_object
+ .get_forwarded_object()
+ .unwrap_or(target_object)
+ };
+
+ gc_tls
+ .object_closure
+ .set_temporarily_and_run_code(visit_object, || {
+ self.process_weak_references(true);
+ })
+ } else {
+ self.process_weak_references(false);
+ }
}
}
-struct ProcessWeakReferences;
-
-impl GCWork<Ruby> for ProcessWeakReferences {
- fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+impl ProcessWeakReferences {
+ fn process_weak_references(&mut self, moving_gc: bool) {
let mut weak_references = crate::binding()
.weak_proc
.weak_references
.try_lock()
.expect("Mutators should not be holding the lock.");
- for ptr_ptr in weak_references.iter_mut() {
- if !(**ptr_ptr).is_reachable() {
- **ptr_ptr = crate::binding().weak_reference_dead_value;
- }
+ weak_references.retain_mut(|object_ptr| {
+ let object = object_ptr.get_forwarded_object().unwrap_or(*object_ptr);
+
+ if object != *object_ptr {
+ *object_ptr = object;
}
- weak_references.clear();
+ if object.is_reachable() {
+ (upcalls().handle_weak_references)(object, moving_gc);
+
+ true
+ } else {
+ false
+ }
+ });
}
}
@@ -165,11 +245,10 @@ trait GlobalTableProcessingWork {
let forward_object = |_worker, object: ObjectReference, _pin| {
debug_assert!(
mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(),
- "{} is not an MMTk object",
- object
+ "{object} is not an MMTk object"
);
let result = object.forward();
- trace!("Forwarding reference: {} -> {}", object, result);
+ trace!("Forwarding reference: {object} -> {result}");
result
};
@@ -185,7 +264,6 @@ struct UpdateFinalizerObjIdTables;
impl GlobalTableProcessingWork for UpdateFinalizerObjIdTables {
fn process_table(&mut self) {
(crate::upcalls().update_finalizer_table)();
- (crate::upcalls().update_obj_id_tables)();
}
}
impl GCWork<Ruby> for UpdateFinalizerObjIdTables {
@@ -195,11 +273,14 @@ impl GCWork<Ruby> for UpdateFinalizerObjIdTables {
}
struct UpdateGlobalTables {
- idx: i32
+ idx: i32,
}
impl GlobalTableProcessingWork for UpdateGlobalTables {
fn process_table(&mut self) {
- (crate::upcalls().update_global_tables)(self.idx)
+ (crate::upcalls().update_global_tables)(
+ self.idx,
+ crate::mmtk().get_plan().current_gc_may_move_object(),
+ )
}
}
impl GCWork<Ruby> for UpdateGlobalTables {
@@ -224,14 +305,10 @@ impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList {
if object.is_reachable() {
// Forward and add back to the candidate list.
let new_object = object.forward();
- trace!(
- "Forwarding WB-unprotected object: {} -> {}",
- object,
- new_object
- );
+ trace!("Forwarding WB-unprotected object: {object} -> {new_object}");
objects.insert(new_object);
} else {
- trace!("Removing WB-unprotected object from list: {}", object);
+ trace!("Removing WB-unprotected object from list: {object}");
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-04 00:33:38 +0000