bin/bm

   1 #!/usr/bin/env ruby
   2
   3 # Must use our benchmark.rb, which has bug fixes -vs- Ruby 1.8.6 pl 111
   4 require File.expand_path(File.join(File.dirname(__FILE__),'../lib/benchmark'))
   5 include Benchmark
   6
   7 # Options handling
   8 require 'optparse'
   9 require 'ostruct'
  10
  11 # Benchmark test selector
  12 # BM_GLOB        = 'test/benchmark/*/bm*.rb'
  13 BM_GLOB        = File.expand_path(File.join(File.dirname(__FILE__),'../benchmark/*/bm*.rb'))
  14
  15 # Which Ruby runtimes to test: MRI should be first and Rubinius should be last
  16 $interpreters       = ['ruby','shotgun/rubinius']
  17
  18 # WIDTH of test labels, required for nice formating
  19 WIDTH          = 50
  20
  21 REPORT_WIDTH   = WIDTH - 6
  22 COLUMN_WIDTH   = 12
  23
  24 DECIMAL_FORMAT = '%4.4f'
  25
  26 now            = Time.now
  27 NOW            = "#{now.year}-#{'%02d' % now.month}-#{'%02d' % now.day}-" +
  28                  "#{'%02d' % now.hour}-#{'%02d' % now.min}-#{'%02d' % now.sec}"
  29
  30 OPTIONS = OpenStruct.new( :size   => 1024,
  31                           :path   => './' )
  32
  33 options = OptionParser.new do |opts|
  34   opts.on('-c', '--chart [FILENAME]', 'Output passing test times to SVG file at --path') do |c|
  35     OPTIONS.chart = c || "#{NOW}-times.svg"
  36   end
  37
  38   opts.on('-d', '--diff-chart [FILENAME]', 'Output speed differences to SVG file at --path') do |d|
  39     OPTIONS.diffchart = d || "#{NOW}-difference.svg"
  40   end
  41
  42   opts.on('-f', '--file [FILENAME]', 'Output report to text file at --path, instead of STDOUT') do |f|
  43     OPTIONS.file = f || "#{NOW}-report.txt"
  44   end
  45
  46   opts.on('-g', '--grep REGEX', 'Only run tests that match REGEX') do |g|
  47     OPTIONS.grep = g
  48   end
  49
  50   opts.on("-t", "--target TARGET", String,
  51           "Use TARGET to compare against: r:ruby|r19:ruby19|x:rbx|j:jruby") do |t|
  52     case t
  53     when 'r', 'ruby'
  54       target = 'ruby'
  55     when 'r19', 'ruby19'
  56       target = 'ruby19'
  57     when 'x', 'rbx', 'rubinius'
  58       target = 'shotgun/rubinius'
  59     when 'j', 'jruby'
  60       target = 'jruby'
  61     else
  62       target = t
  63     end
  64     $interpreters.insert(1,target)
  65   end
  66
  67   opts.on('-j', '--just', 'Benchmark just the target machine') do |j|
  68     $interpreters.delete_at(0)
  69   end
  70
  71   opts.on('-h', '--help', 'Output this help information') do |h|
  72     OPTIONS.help = h
  73   end
  74
  75   opts.on('-n', '--ngrep REGEX', 'Only run tests that do not match REGEX') do |n|
  76     OPTIONS.ngrep = n
  77   end
  78
  79   opts.on('-p', '--path PATH', 'Path prefix for --chart, --diff-chart, and --file') do |p|
  80     OPTIONS.path = p
  81   end
  82
  83   opts.on('-s', '--size PIXELS', 'Size, in pixels, of --chart and/or --diff-chart') do |s|
  84     OPTIONS.size = s
  85   end
  86
  87   opts.on('-w', '--warmup', 'Run each test twice and throw away the first result') do |w|
  88     OPTIONS.warmup = w
  89   end
  90 end
  91
  92 options.parse!
  93
  94 if OPTIONS.help then
  95   puts options.summarize
  96   exit
  97 end
  98
  99 class RbxBenchmarkFile
 100   def initialize(interpreter,file)
 101     @interpreter = interpreter
 102     @file        = file
 103   end
 104
 105   def pretty_interpreter
 106     @interpreter.match(%r{ruby19|rubinius|ruby|jruby})[0]
 107   end
 108
 109   def pretty_file
 110     @file.match(%r{[^/]+/[^/]+$})[0]
 111   end
 112
 113   def label
 114     pretty_interpreter + ': ' + pretty_file
 115   end
 116
 117   def command
 118     command = "#{@interpreter} #{@file}"
 119
 120     case @file
 121       when /borasky\/bm_MatrixBenchmark[.]rb/
 122         command << %{ 64}
 123     end
 124
 125     command
 126   end
 127
 128   def quiet_command
 129     command + ' > /dev/null 2>&1'
 130   end
 131
 132   def run
 133     spaces = WIDTH - label.length
 134     spaces = spaces < 0 ? 0 : spaces
 135
 136     @result = if OPTIONS.warmup then
 137                 Benchmark.bmbm do |x|
 138                   x.report(label + ' ' * spaces) do
 139                     if system quiet_command then
 140                       @pass = true
 141                     else
 142                       @pass = false
 143                       system command
 144                     end
 145                   end
 146                 end.first
 147               else
 148                 Benchmark.bm do |x|
 149                   x.report(label + ' ' * spaces) do
 150                     if system quiet_command then
 151                       @pass = true
 152                     else
 153                       @pass = false
 154                       system command
 155                     end
 156                   end
 157                 end
 158               end
 159   end
 160
 161   def passed?
 162     @pass
 163   end
 164
 165   def failed?
 166     ! @pass
 167   end
 168
 169   def time
 170     passed? ? @result.real : 0
 171   end
 172
 173   def skip
 174     @skipped = true
 175   end
 176
 177   def skipped?
 178     @skipped
 179   end
 180
 181   def unchartable?
 182     failed? or skipped?
 183   end
 184
 185   def report
 186     if skipped? then
 187       'Skipped'
 188     elsif passed? then
 189       DECIMAL_FORMAT % time
 190     else
 191       'Failed'
 192     end.rjust(COLUMN_WIDTH)
 193   end
 194 end
 195
 196 class RbxBenchmarkRun
 197   attr_reader :interpreter
 198   attr_reader :results
 199
 200   def initialize(interpreter)
 201     @interpreter = interpreter
 202     @results     = []
 203     @total       = Benchmark::Tms.new
 204   end
 205
 206   def clean
 207     if @interpreter =~ /rubinius/ then
 208       Dir[BM_GLOB + 'c'].each do |file|
 209         File.unlink file
 210       end
 211     end
 212   end
 213
 214   def run
 215     clean
 216
 217     Dir[BM_GLOB].each do |file|
 218       @results << RbxBenchmarkFile.new(@interpreter,file)
 219
 220       if OPTIONS.grep and ! file.match(OPTIONS.grep)
 221         @results.last.skip
 222         next
 223       end
 224
 225       if OPTIONS.ngrep and file.match(OPTIONS.ngrep) then
 226         @results.last.skip
 227         next
 228       end
 229
 230       @results.last.run
 231     end
 232
 233     self
 234   end
 235
 236   def time
 237     @results.inject(0) { |total, result| total + result.time }
 238   end
 239
 240   def chart_points
 241     @results.collect do |result|
 242       if result.unchartable?
 243         nil
 244       else
 245         result.time
 246       end
 247     end
 248   end
 249 end
 250
 251 def difference(ruby,rubinius)
 252   if ruby.unchartable? or rubinius.unchartable? then
 253     nil
 254   elsif ruby.time < rubinius.time then
 255     -rubinius.time / ruby.time
 256   else
 257     ruby.time / rubinius.time
 258   end
 259 end
 260
 261 def report(runs)
 262   header = 'file'.ljust(REPORT_WIDTH)
 263
 264   runs.each do |run|
 265     header << (run.results.first.pretty_interpreter).rjust(COLUMN_WIDTH)
 266   end
 267
 268   header << 'difference'.rjust(COLUMN_WIDTH)
 269
 270   file = nil
 271
 272   if OPTIONS.file
 273     file = File.open(OPTIONS.path + OPTIONS.file,'w')
 274   else
 275     file = STDOUT
 276   end
 277
 278   file.puts header
 279   file.puts '-' * header.length
 280
 281   # Columnar result output
 282   0.upto(runs.first.results.length - 1) do |i|
 283     line = runs.first.results[i].pretty_file.ljust(REPORT_WIDTH)
 284     results = []
 285     runs.each do |run|
 286       results << run.results[i]
 287       line    << run.results[i].report
 288     end
 289
 290     diff = difference(results.first,results.last)
 291
 292     line << if diff.nil? then
 293               'n/a'
 294             else
 295               DECIMAL_FORMAT % diff
 296             end.rjust(COLUMN_WIDTH)
 297
 298     file.puts line
 299   end
 300
 301   file.close unless file == STDOUT
 302 end
 303
 304 runs = []
 305
 306 $interpreters.each do |i|
 307   runs << RbxBenchmarkRun.new(i).run
 308 end
 309
 310 report(runs)
 311
 312 if OPTIONS.chart or OPTIONS.diffchart then
 313   require 'rubygems'
 314   require 'scruffy'
 315 end
 316
 317 if OPTIONS.chart then
 318   chart_point_sets = runs.collect { |run| run.chart_points }
 319
 320   # Make a list of failed tests
 321   deletes = []
 322
 323   for i in 0..chart_point_sets.length-1
 324     delete = false
 325     for j in 0..chart_point_sets.first.length-1
 326       if chart_point_sets[i][j].nil? then
 327         deletes << j
 328       end
 329     end
 330   end
 331
 332   # Remove failed tests from each runtime
 333   chart_point_sets.each do |cps|
 334     deletes.uniq.sort.reverse.each { |d| cps.delete_at(d) }
 335   end
 336
 337   graph = Scruffy::Graph.new
 338
 339   runs.each do |run|
 340     graph << Scruffy::Layers::Line.new( :title  => run.interpreter,
 341                                         :points => chart_point_sets.shift )
 342   end
 343
 344   # graph.point_markers = runs.first.results.reject do |result|
 345   #   result.unchartable?
 346   # end.collect do |result|
 347   #   result.pretty_file
 348   # end
 349
 350   puts "Rendering chart to #{OPTIONS.chart}"
 351
 352   graph.render( :width => OPTIONS.size, :to => OPTIONS.path + OPTIONS.chart )
 353 end
 354
 355 if OPTIONS.diffchart then
 356
 357   differences = []
 358
 359   0.upto(runs.first.results.length - 1) do |i|
 360     differences << difference(runs.first.results[i],runs.last.results[i])
 361   end
 362
 363   # Remove where tests failed, easier this time, only a single list
 364   differences.reject! { |d| d.nil? }
 365
 366   # Convert to percentages
 367   # differences.collect { |d| d * 100 }
 368
 369   graph = Scruffy::Graph.new
 370
 371   graph << Scruffy::Layers::Line.new( :title  => 'Zero',
 372                                       :points => differences.collect { |d| 0 } )
 373
 374   graph << Scruffy::Layers::Line.new( :title  => 'Rubinius times faster (-times slower)',
 375                                       :points => differences )
 376
 377   # graph.point_markers = runs.first.results.reject do |result|
 378   #   result.unchartable?
 379   # end.collect do |result|
 380   #   result.pretty_file
 381   # end
 382
 383   puts "Rendering diffchart to #{OPTIONS.diffchart}"
 384
 385   graph.render( :width => OPTIONS.size, :to => OPTIONS.path + OPTIONS.diffchart )
 386 end