summaryrefslogtreecommitdiff
path: root/sample
diff options
context:
space:
mode:
authorko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2015-12-08 13:58:50 (GMT)
committerko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2015-12-08 13:58:50 (GMT)
commit3dbb390180a0e9f98623b6db0d71b0213359c541 (patch)
treead18953723a41185b127a340386427b3c86d6904 /sample
parent8f620b9b17ccbaae1e3eb9a1b9b5d528f4d5f900 (diff)
* introduce new ISeq binary format serializer/de-serializer
and a pre-compilation/runtime loader sample. [Feature #11788] * iseq.c: add new methods: * RubyVM::InstructionSequence#to_binary_format(extra_data = nil) * RubyVM::InstructionSequence.from_binary_format(binary) * RubyVM::InstructionSequence.from_binary_format_extra_data(binary) * compile.c: implement body of this new feature. * load.c (rb_load_internal0), iseq.c (rb_iseq_load_iseq): call RubyVM::InstructionSequence.load_iseq(fname) with loading script name if this method is defined. We can return any ISeq object as a result value. Otherwise loading will be continue as usual. This interface is not matured and is not extensible. So that we don't guarantee the future compatibility of this method. Basically, you should'nt use this method. * iseq.h: move ISEQ_MAJOR/MINOR_VERSION (and some definitions) from iseq.c. * encoding.c (rb_data_is_encoding), internal.h: added. * vm_core.h: add several supports for lazy load. * add USE_LAZY_LOAD macro to specify enable or disable of this feature. * add several fields to rb_iseq_t. * introduce new macro rb_iseq_check(). * insns.def: some check for lazy loading feature. * vm_insnhelper.c: ditto. * proc.c: ditto. * vm.c: ditto. * test/lib/iseq_loader_checker.rb: enabled iff suitable environment variables are provided. * test/runner.rb: enable lib/iseq_loader_checker.rb. * sample/iseq_loader.rb: add sample compiler and loader. $ ruby sample/iseq_loader.rb [dir] will compile all ruby scripts in [dir]. With default setting, this compile creates *.rb.yarb files in same directory of target .rb scripts. $ ruby -r sample/iseq_loader.rb [app] will run with enable to load compiled binary data. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@52949 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'sample')
-rw-r--r--sample/iseq_loader.rb240
1 files changed, 240 insertions, 0 deletions
diff --git a/sample/iseq_loader.rb b/sample/iseq_loader.rb
new file mode 100644
index 0000000..4fbf02b
--- /dev/null
+++ b/sample/iseq_loader.rb
@@ -0,0 +1,240 @@
+#
+# iseq_loader.rb - sample of compiler/loader for binary compiled file
+#
+# Usage as a compiler: ruby iseq_loader.rb [file or directory] ...
+#
+# It compiles and stores specified files.
+# If directories are specified, then compiles and stores all *.rb files.
+# (using Dir.glob)
+#
+# TODO: add remove option
+# TODO: add verify option
+#
+# Usage as a loader: simply require this file with the following setting.
+#
+# Setting with environment variables.
+#
+# * RUBY_ISEQ_LOADER_STORAGE to select storage type
+# * dbm: use dbm
+# * fs: [default] use file system. locate a compiled binary files in same
+# directory of scripts like Rubinius. foo.rb.yarb will be created for foo.rb.
+# * fs2: use file system. locate compiled file in specified directory.
+# * nothing: do nothing.
+#
+# * RUBY_ISEQ_LOADER_STORAGE_DIR to select directory
+# * default: ~/.ruby_binaries/
+#
+# * RUBY_ISEQ_LOADER_STORAGE_COMPILE_IF_NOT_COMPILED
+# * true: store compiled file if compiled data is not available.
+# * false: [default] do nothing if there is no compiled iseq data.
+
+class RubyVM::InstructionSequence
+ $ISEQ_LOADER_LOADED = 0
+ $ISEQ_LOADER_COMPILED = 0
+ $ISEQ_LOADER_IGNORED = 0
+ LAUNCHED_TIME = Time.now
+ COMPILE_FILE_ENABLE = false || true
+ COMPILE_VERBOSE = $VERBOSE || false # || true
+ COMPILE_DEBUG = ENV['RUBY_ISEQ_LOADER_DEBUG']
+ COMPILE_IF_NOT_COMPILED = ENV['RUBY_ISEQ_LOADER_STORAGE_COMPILE_IF_NOT_COMPILED'] == 'true'
+
+ at_exit{
+ STDERR.puts "[ISEQ_LOADER] #{Process.pid} time: #{Time.now - LAUNCHED_TIME}, " +
+ "loaded: #{$ISEQ_LOADER_LOADED}, " +
+ "compied: #{$ISEQ_LOADER_COMPILED}, " +
+ "ignored: #{$ISEQ_LOADER_IGNORED}"
+ } if COMPILE_VERBOSE
+
+ unless cf_dir = ENV['RUBY_ISEQ_LOADER_STORAGE_DIR']
+ cf_dir = File.expand_path("~/.ruby_binaries")
+ unless File.exist?(cf_dir)
+ Dir.mkdir(cf_dir)
+ end
+ end
+ CF_PREFIX = "#{cf_dir}/cb."
+
+ class NullStorage
+ def load_iseq fname; end
+ def compile_and_save_isq fname; end
+ def unlink_compiled_iseq; end
+ end
+
+ class BasicStorage
+ def initialize
+ require 'digest/sha1'
+ end
+
+ def load_iseq fname
+ iseq_key = iseq_key_name(fname)
+ if compiled_iseq_exist?(fname, iseq_key) && compiled_iseq_is_younger?(fname, iseq_key)
+ $ISEQ_LOADER_LOADED += 1
+ STDERR.puts "[ISEQ_LOADER] #{Process.pid} load #{fname} from #{iseq_key}" if COMPILE_DEBUG
+ binary = read_compiled_iseq(fname, iseq_key)
+ RubyVM::InstructionSequence.from_binary_format(binary)
+ elsif COMPILE_IF_NOT_COMPILED
+ compile_and_save_iseq(fname, iseq_key)
+ else
+ $ISEQ_LOADER_IGNORED += 1
+ # p fname
+ nil
+ end
+ end
+
+ def extra_data fname
+ "SHA-1:#{::Digest::SHA1.file(fname).digest}"
+ end
+
+ def compile_and_save_iseq fname, iseq_key = iseq_key_name(fname)
+ $ISEQ_LOADER_COMPILED += 1
+ STDERR.puts "[RUBY_COMPILED_FILE] compile #{fname}" if COMPILE_DEBUG
+ iseq = RubyVM::InstructionSequence.compile_file(fname)
+
+ binary = iseq.to_binary_format(extra_data(fname))
+ write_compiled_iseq(fname, iseq_key, binary)
+ iseq
+ end
+
+ # def unlink_compiled_iseq; nil; end # should implement at sub classes
+
+ private
+
+ def iseq_key_name fname
+ fname
+ end
+
+ # should implement at sub classes
+ # def compiled_iseq_younger? fname, iseq_key; end
+ # def compiled_iseq_exist? fname, iseq_key; end
+ # def read_compiled_file fname, iseq_key; end
+ # def write_compiled_file fname, iseq_key, binary; end
+ end
+
+ class FSStorage < BasicStorage
+ def initialize
+ super
+ require 'fileutils'
+ @dir = CF_PREFIX + "files"
+ unless File.directory?(@dir)
+ FileUtils.mkdir_p(@dir)
+ end
+ end
+
+ def unlink_compiled_iseq
+ File.unlink(compile_file_path)
+ end
+
+ private
+
+ def iseq_key_name fname
+ "#{fname}.yarb" # same directory
+ end
+
+ def compiled_iseq_exist? fname, iseq_key
+ File.exist?(iseq_key)
+ end
+
+ def compiled_iseq_is_younger? fname, iseq_key
+ File.mtime(iseq_key) >= File.mtime(fname)
+ end
+
+ def read_compiled_iseq fname, iseq_key
+ open(iseq_key, 'rb'){|f| f.read}
+ end
+
+ def write_compiled_iseq fname, iseq_key, binary
+ open(iseq_key, 'wb'){|f| f.write(binary)}
+ end
+ end
+
+ class FS2Storage < FSStorage
+ def iseq_key_name fname
+ @dir + fname.gsub(/[^A-Za-z0-9\._-]/){|c| '%02x' % c.ord} # special directory
+ end
+ end
+
+ class DBMStorage < BasicStorage
+ def initialize
+ require 'dbm'
+ @db = DBM.open(CF_PREFIX+'db')
+ end
+
+ def unlink_compiled_iseq
+ @db.delete fname
+ end
+
+ private
+
+ def date_key_name fname
+ "date.#{fname}"
+ end
+
+ def iseq_key_name fname
+ "body.#{fname}"
+ end
+
+ def compiled_iseq_exist? fname, iseq_key
+ @db.has_key? iseq_key
+ end
+
+ def compiled_iseq_is_younger? fname, iseq_key
+ date_key = date_key_name(fname)
+ if @db.has_key? date_key
+ @db[date_key].to_i >= File.mtime(fname).to_i
+ end
+ end
+
+ def read_compiled_iseq fname, iseq_key
+ @db[iseq_key]
+ end
+
+ def write_compiled_iseq fname, iseq_key, binary
+ date_key = date_key_name(fname)
+ @db[iseq_key] = binary
+ @db[date_key] = Time.now.to_i
+ end
+ end
+
+ STORAGE = case ENV['RUBY_ISEQ_LOADER_STORAGE']
+ when 'dbm'
+ DBMStorage.new
+ when 'fs'
+ FSStorage.new
+ when 'fs2'
+ FS2Storage.new
+ when 'null'
+ NullStorage.new
+ else
+ FSStorage.new
+ end
+
+ STDERR.puts "[ISEQ_LOADER] use #{STORAGE.class} " if COMPILE_VERBOSE
+
+ def self.load_iseq fname
+ STORAGE.load_iseq(fname)
+ end
+
+ def self.compile_and_save_iseq fname
+ STORAGE.compile_and_save_iseq fname
+ end
+
+ def self.unlink_compiled_iseq fname
+ STORAGE.unlink_compiled_iseq fname
+ end
+end
+
+if __FILE__ == $0
+ ARGV.each{|path|
+ if File.directory?(path)
+ pattern = File.join(path, '**/*.rb')
+ Dir.glob(pattern){|file|
+ begin
+ RubyVM::InstructionSequence.compile_and_save_iseq(file)
+ rescue SyntaxError => e
+ STDERR.puts e
+ end
+ }
+ else
+ RubyVM::InstructionSequence.compile_and_save_iseq(path)
+ end
+ }
+end