summaryrefslogtreecommitdiff
path: root/tool/transform_mjit_header.rb
blob: 218023682470391084a372a7985031f0db5eb52f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com>
# This is a script to transform functions to static inline.
# Usage: transform_mjit_header.rb <c-compiler> <header file> <out>

require 'fileutils'
require 'tempfile'

PROGRAM = File.basename($0, ".*")

module MJITHeader
  ATTR_VALUE_REGEXP  = /[^()]|\([^()]*\)/
  ATTR_REGEXP        = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/
  # Example:
  #   VALUE foo(int bar)
  #   VALUE __attribute__ ((foo)) bar(int baz)
  #   __attribute__ ((foo)) VALUE bar(int baz)
  FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/
  TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/

  # Predefined macros for compilers which are already supported by MJIT.
  # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro.
  SUPPORTED_CC_MACROS = [
    '__GNUC__', # gcc
    '__clang__', # clang
  ]

  # These macros are relied on this script's transformation
  PREFIXED_MACROS = [
    'ALWAYS_INLINE',
    'COLDFUNC',
    'inline',
    'RBIMPL_ATTR_COLD',
  ]

  # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times.
  RECURSIVE_MACROS = %w[
    RASCTRYINFO
    RASIPADDR
  ]

  IGNORED_FUNCTIONS = [
    'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function.
    'rb_equal_opt', # Not used from VM and not compilable
    'ruby_abi_version',
  ]

  ALWAYS_INLINED_FUNCTIONS = [
    'vm_opt_plus',
    'vm_opt_minus',
    'vm_opt_mult',
    'vm_opt_div',
    'vm_opt_mod',
    'vm_opt_neq',
    'vm_opt_lt',
    'vm_opt_le',
    'vm_opt_gt',
    'vm_opt_ge',
    'vm_opt_ltlt',
    'vm_opt_and',
    'vm_opt_or',
    'vm_opt_aref',
    'vm_opt_aset',
    'vm_opt_aref_with',
    'vm_opt_aset_with',
    'vm_opt_not',
  ]

  COLD_FUNCTIONS = %w[
    setup_parameters_complex
    vm_call_iseq_setup
    vm_call_iseq_setup_2
    vm_call_iseq_setup_tailcall
    vm_call_method_each_type
    vm_ic_update
  ]

  # Return start..stop of last decl in CODE ending STOP
  def self.find_decl(code, stop)
    level = 0
    i = stop
    while i = code.rindex(/[;{}]/, i)
      if level == 0 && stop != i && decl_found?($&, i)
        return decl_start($&, i)..stop
      end
      case $&
      when '}'
        level += 1
      when '{'
        level -= 1
      end
      i -= 1
    end
    nil
  end

  def self.decl_found?(code, i)
    i == 0 || code == ';' || code == '}'
  end

  def self.decl_start(code, i)
    if i == 0 && code != ';' && code != '}'
      0
    else
      i + 1
    end
  end

  # Given DECL return the name of it, nil if failed
  def self.decl_name_of(decl)
    ident_regex = /\w+/
    decl = decl.gsub(/^#.+$/, '') # remove macros
    reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes
    su1_regex = /{[^{}]*}/
    su2_regex = /{([^{}]|#{su1_regex})*}/
    su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough
    reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header
    id_seq_regex = /\s*(?:#{ident_regex}(?:\s+|\s*[*]+\s*))*/
    # Process function header:
    match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl)
    return match[:name] if match
    # Process non-function declaration:
    reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization
    match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl);
    return match[:name] if match
    nil
  end

  # Return true if CC with CFLAGS compiles successfully the current code.
  # Use STAGE in the message in case of a compilation failure
  def self.check_code!(code, cc, cflags, stage)
    with_code(code) do |path|
      cmd = "#{cc} #{cflags} #{path}"
      out = IO.popen(cmd, err: [:child, :out], &:read)
      unless $?.success?
        STDERR.puts "error in #{stage} header file:\n#{out}"
        exit false
      end
    end
  end

  # Remove unpreprocessable macros
  def self.remove_harmful_macros!(code)
    code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '')
  end

  # -dD outputs those macros, and it produces redefinition warnings or errors
  # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h.
  def self.remove_predefined_macros!(code)
    code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2')
  end

  # Return [macro, others]. But others include PREFIXED_MACROS to be used in code.
  def self.separate_macro_and_code(code)
    code.lines.partition do |l|
      l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") }
    end.map! { |lines| lines.join('') }
  end

  def self.write(code, out)
    # create with strict permission, then will install proper
    # permission
    FileUtils.mkdir_p(File.dirname(out), mode: 0700)
    File.binwrite("#{out}.new", code, perm: 0600)
    FileUtils.mv("#{out}.new", out)
  end

  # Note that this checks runruby. This conservatively covers platform names.
  def self.windows?
    RUBY_PLATFORM =~ /mswin|mingw|msys/
  end

  # If code has macro which only supported compilers predefine, return true.
  def self.supported_header?(code)
    SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ }
  end

  # This checks if syntax check outputs one of the following messages.
  #    "error: conflicting types for 'restrict'"
  #    "error: redefinition of parameter 'restrict'"
  # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround.
  def self.conflicting_types?(code, cc, cflags)
    with_code(code) do |path|
      cmd = "#{cc} #{cflags} #{path}"
      out = IO.popen(cmd, err: [:child, :out], &:read)
      !$?.success? &&
        (out.match?(/error: conflicting types for '[^']+'/) ||
         out.match?(/error: redefinition of parameter '[^']+'/))
    end
  end

  def self.with_code(code)
    # for `system_header` pragma which can't be in the main file.
    Tempfile.open(['', '.h'], mode: File::BINARY) do |f|
      f.puts code
      f.close
      Tempfile.open(['', '.c'], mode: File::BINARY) do |c|
        c.puts <<SRC
#include "#{f.path}"
SRC
        c.close
        return yield(c.path)
      end
    end
  end
  private_class_method :with_code
end

if ARGV.size != 3
  abort "Usage: #{$0} <c-compiler> <header file> <out>"
end

if STDOUT.tty?
  require_relative 'lib/colorize'
  color = Colorize.new
end
cc      = ARGV[0]
code    = File.binread(ARGV[1]) # Current version of the header file.
outfile = ARGV[2]
cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors'

if !MJITHeader.supported_header?(code)
  puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header."
  MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile)
  exit
end

MJITHeader.remove_predefined_macros!(code)

if MJITHeader.windows? # transformation is broken on Windows and the platform is not supported
  MJITHeader.remove_harmful_macros!(code)
  MJITHeader.check_code!(code, cc, cflags, 'initial')
  puts "\nSkipped transforming external functions to static on Windows."
  MJITHeader.write(code, outfile)
  exit
end

macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW
code = <<header + code
#ifdef __GNUC__
# pragma GCC system_header
#endif
header
code_to_check = "#{code}#{macro}" # macro should not affect code again

if MJITHeader.conflicting_types?(code_to_check, cc, cflags)
  cflags = "#{cflags} -std=c99" # For AIX gcc
end

# Check initial file correctness in the manner of final output.
MJITHeader.check_code!(code_to_check, cc, cflags, 'initial')

stop_pos       = -1
extern_names   = []
transform_logs = Hash.new { |h, k| h[k] = [] }

# This loop changes function declarations to static inline.
while (decl_range = MJITHeader.find_decl(code, stop_pos))
  stop_pos = decl_range.begin - 1
  decl = code[decl_range]
  decl_name = MJITHeader.decl_name_of(decl)

  if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)
    transform_logs[:def_to_decl] << decl_name
    code[decl_range] = decl.sub(/{.+}/m, ';')
  elsif MJITHeader::COLD_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)
    header = match[0].sub(/{\z/, '').strip
    header = "static #{header.sub(/\A((static|inline) )+/, '')}"
    decl[match.begin(0)...match.end(0)] = '{' # remove header
    code[decl_range] = "\nCOLDFUNC #{header} #{decl}"
  elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)
    header = match[0].sub(/{\z/, '').strip
    header = "static inline #{header.sub(/\A((static|inline) )+/, '')}"
    decl[match.begin(0)...match.end(0)] = '{' # remove header
    code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}"
  elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/)
    decl.sub!(/(extern|static|inline) /, ' ')
    unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
      transform_logs[:static_inline_decl] << decl_name
    end

    code[decl_range] = "static inline #{decl}"
  elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/
    unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP)
      transform_logs[:skipped] << decl_name
      next
    end

    extern_names << decl_name
    decl[match.begin(0)...match.end(0)] = ''

    if decl =~ /\bstatic\b/
      abort "#{PROGRAM}: a static decl was found inside external definition #{decl_name.dump}"
    end

    header.sub!(/(extern|inline) /, ' ')
    unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
      transform_logs[:static_inline_def] << decl_name
    end
    code[decl_range] = "static inline #{header}#{decl}"
  end
end

code << macro

# Check the final file correctness
MJITHeader.check_code!(code, cc, cflags, 'final')

MJITHeader.write(code, outfile)

messages = {
  def_to_decl: 'changing definition to declaration',
  static_inline_def: 'making external definition static inline',
  static_inline_decl: 'making declaration static inline',
  skipped: 'SKIPPED to transform',
}
transform_logs.each do |key, decl_names|
  decl_names = decl_names.map { |s| color.bold(s) } if color
  puts("#{PROGRAM}: #{messages.fetch(key)}: #{decl_names.join(', ')}")
end