win32.c: acp_to_wstr results check

* win32/win32.c (rb_w32_spawn, rb_w32_aspawn_flags): check the results
  of acp_to_wstr() which can return NULL.  [ruby-core:51557] [Bug #7721]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38904 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 16 insertions, 8 deletions

diff --git a/win32/win32.c b/win32/win32.c
index 3b827662a7..8b577eaf84 100644
--- a/win32/win32.c
+++ b/win32/win32.c
@@ -1182,7 +1182,8 @@ rb_w32_spawn(int mode, const char *cmd, const char *prog)
     char fbuf[MAXPATHLEN];
     char *p = NULL;
     const char *shell = NULL;
-    WCHAR *wcmd, *wshell;
+    WCHAR *wcmd = NULL, *wshell = NULL;
+    int e = 0;
     rb_pid_t ret;
     VALUE v = 0;
     VALUE v2 = 0;
@@ -1267,14 +1268,17 @@ rb_w32_spawn(int mode, const char *cmd, const char *prog)
     }
 
     /* assume ACP */
-    wcmd = cmd ? acp_to_wstr(cmd, NULL) : NULL;
+    if (!e && cmd && !(wcmd = acp_to_wstr(cmd, NULL))) e = E2BIG;
     if (v) ALLOCV_END(v);
-    wshell = shell ? acp_to_wstr(shell, NULL) : NULL;
+    if (!e && shell && !(wshell = acp_to_wstr(shell, NULL))) e = E2BIG;
     if (v2) ALLOCV_END(v2);
 
-    ret = child_result(CreateChild(wcmd, wshell, NULL, NULL, NULL, NULL, 0), mode);
+    if (!e) {
+	ret = child_result(CreateChild(wcmd, wshell, NULL, NULL, NULL, NULL, 0), mode);
+    }
     free(wshell);
     free(wcmd);
+    if (e) errno = e;
     return ret;
 }
 
@@ -1287,7 +1291,8 @@ rb_w32_aspawn_flags(int mode, const char *prog, char *const *argv, DWORD flags)
     BOOL ntcmd = FALSE, tmpnt;
     const char *shell;
     char *cmd, fbuf[MAXPATHLEN];
-    WCHAR *wcmd, *wprog;
+    WCHAR *wcmd = NULL, *wprog = NULL;
+    int e = 0;
     rb_pid_t ret;
     VALUE v = 0;
 
@@ -1335,13 +1340,16 @@ rb_w32_aspawn_flags(int mode, const char *prog, char *const *argv, DWORD flags)
     }
 
     /* assume ACP */
-    wcmd = cmd ? acp_to_wstr(cmd, NULL) : NULL;
+    if (!e && cmd && !(wcmd = acp_to_wstr(cmd, NULL))) e = E2BIG;
     if (v) ALLOCV_END(v);
-    wprog = prog ? acp_to_wstr(prog, NULL) : NULL;
+    if (!e && prog && !(wprog = acp_to_wstr(prog, NULL))) e = E2BIG;
 
-    ret = child_result(CreateChild(wcmd, wprog, NULL, NULL, NULL, NULL, flags), mode);
+    if (!e) {
+	ret = child_result(CreateChild(wcmd, wprog, NULL, NULL, NULL, NULL, flags), mode);
+    }
     free(wprog);
     free(wcmd);
+    if (e) errno = e;
     return ret;
 }