thread_pthread.c: allocate sigaltstack before pthread_create

A new (not-initialized-yet) pthread attempts to allocate sigaltstack by
using xmalloc.  It may cause GC, but because the thread is not
initialized yet, ruby_native_thread_p() returns false, which leads to
"[FATAL] failed to allocate memory" and exit.

In fact, we can observe the error message in the log of OpenBSD CI:
https://rubyci.org/logs/rubyci.s3.amazonaws.com/openbsd-current/ruby-master/log/20200306T083005Z.log.html.gz

This changeset allocates sigaltstack before pthread is created.

3 files changed, 27 insertions, 8 deletions

diff --git a/signal.c b/signal.c
index ab4eadc7ee..650c59a084 100644
--- a/signal.c
+++ b/signal.c
@@ -547,14 +547,25 @@ rb_sigaltstack_size(void)
     return size;
 }
 
+static int rb_sigaltstack_size_value = 0;
+
+void *
+rb_allocate_sigaltstack(void)
+{
+    if (!rb_sigaltstack_size_value) {
+	rb_sigaltstack_size_value = rb_sigaltstack_size();
+    }
+    return xmalloc(rb_sigaltstack_size_value);
+}
+
 /* alternate stack for SIGSEGV */
 void *
-rb_register_sigaltstack(void)
+rb_register_sigaltstack(void *altstack)
 {
     stack_t newSS, oldSS;
 
-    newSS.ss_size = rb_sigaltstack_size();
-    newSS.ss_sp = xmalloc(newSS.ss_size);
+    newSS.ss_size = rb_sigaltstack_size_value;
+    newSS.ss_sp = altstack;
     newSS.ss_flags = 0;
 
     sigaltstack(&newSS, &oldSS); /* ignore error. */
@@ -1561,7 +1572,7 @@ Init_signal(void)
 	force_install_sighandler(SIGILL, (sighandler_t)sigill, &default_sigill_handler);
 #endif
 #ifdef SIGSEGV
-	RB_ALTSTACK_INIT(GET_VM()->main_altstack);
+	RB_ALTSTACK_INIT(GET_VM()->main_altstack, rb_allocate_sigaltstack());
 	force_install_sighandler(SIGSEGV, (sighandler_t)sigsegv, &default_sigsegv_handler);
 #endif
     }
diff --git a/thread_pthread.c b/thread_pthread.c
index 29284ff0f9..b9ac63f298 100644
--- a/thread_pthread.c
+++ b/thread_pthread.c
@@ -950,7 +950,7 @@ static void *
 thread_start_func_1(void *th_ptr)
 {
     rb_thread_t *th = th_ptr;
-    RB_ALTSTACK_INIT(void *altstack);
+    RB_ALTSTACK_INIT(void *altstack, th->altstack);
 #if USE_THREAD_CACHE
   thread_start:
 #endif
@@ -1099,6 +1099,9 @@ native_thread_create(rb_thread_t *th)
         const size_t stack_size = th->vm->default_params.thread_machine_stack_size + th->vm->default_params.thread_vm_stack_size;
 	const size_t space = space_size(stack_size);
 
+#ifdef USE_SIGALTSTACK
+        th->altstack = rb_allocate_sigaltstack();
+#endif
         th->ec->machine.stack_maxsize = stack_size - space;
 
 	CHECK_ERR(pthread_attr_init(&attr));
diff --git a/vm_core.h b/vm_core.h
index d84e05d99b..754ce981e6 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -141,12 +141,13 @@
 
 #if defined(SIGSEGV) && defined(HAVE_SIGALTSTACK) && defined(SA_SIGINFO) && !defined(__NetBSD__)
 #  define USE_SIGALTSTACK
-void *rb_register_sigaltstack(void);
-#  define RB_ALTSTACK_INIT(var) var = rb_register_sigaltstack()
+void *rb_allocate_sigaltstack(void);
+void *rb_register_sigaltstack(void *);
+#  define RB_ALTSTACK_INIT(var, altstack) var = rb_register_sigaltstack(altstack)
 #  define RB_ALTSTACK_FREE(var) xfree(var)
 #  define RB_ALTSTACK(var)  var
 #else /* noop */
-#  define RB_ALTSTACK_INIT(var)
+#  define RB_ALTSTACK_INIT(var, altstack)
 #  define RB_ALTSTACK_FREE(var)
 #  define RB_ALTSTACK(var) (0)
 #endif
@@ -974,6 +975,10 @@ typedef struct rb_thread_struct {
     /* misc */
     VALUE name;
 
+#ifdef USE_SIGALTSTACK
+    void *altstack;
+#endif
+
 } rb_thread_t;
 
 typedef enum {