bin/dmtest - main - githubmirror/device-mapper-test-suite

bin/dmtest
689 строк16 KB
Joe Thornber
Allow 'list' command to load TestOutcome class.
04 янв 2023, 14:59
04 янв 2023, 14:59324037c
#!/usr/bin/env ruby

require 'pp'
require 'erb'
require 'dmtest/command_line'
require 'dmtest/log'
require 'dmtest/report-generators/report_templates'
require 'dmtest/test-outcome'
require 'dmtest/config'
require 'dmtest/maybe_colored'
require 'pathname'

require 'test/unit/collector/objectspace'
require 'test/unit/testsuite'
require 'test/unit/ui/testrunnermediator'
require 'test/unit/ui/testrunnerutilities'

require 'webrick'
require 'yaml'

#----------------------------------------------------------------

# Hacky global variable
$failed = false

#----------------------------------------------------------------

Message = Struct.new(:level, :time, :txt)

class File
  def each_message
    msg = nil
    in_message = false

    each_line do |line|
      if in_message
        m = /([DIWE]), (\[.*\])[^:]*: (.*)/.match(line)
        if m
          yield(msg)
          msg = Message.new(m[1], trim_time(m[2]), m[3])
        else
          msg.txt.concat line
        end
      else
        m = /([DIWE]), (\[.*\])[^:]*: (.*)/.match(line)
        if !m
          raise RuntimeError, "bad log line: '#{line}'"
        end

        msg = Message.new(m[1], trim_time(m[2]), m[3])
        in_message = true
      end
    end

    if in_message
      yield(msg)
    end
  end

  private
  def trim_time(txt)
    # [2011-10-19T15:02:36.011520 #1065]
    m = /T(\S+)/.match(txt)
    m ? m[1] : 'arf'
  end
end

#----------------------------------------------------------------

TestSpecifier = Struct.new(:suite, :name)

# Loads the yaml files for the test outcomes on demand
class OutcomeCache
  def initialize(log_dir)
    @log_dir = log_dir
    @outcomes = {}
  end

  # Returns nil if the yaml file is not present
  def get_outcome(specifier)
    @outcomes[specifier] ||= load(specifier)
  end

  private
  def load(specifier)
    path = "#{@log_dir}/#{specifier.suite}_#{specifier.name}.yaml"
    if File::exist?(path)
      @outcomes[specifier] = File.open(path, 'r:bom|utf-8') {|f|
        input = f.read
        Psych.safe_load(input, :permitted_classes => [TestOutcome, Time])
      }
    else
      nil
    end
  end
end

#----------------------------------------------------------------

# based on the console test runner
module Test
  module Unit
    module UI
      class DMTestMediator
        RESET = name + "::RESET"
        STARTED = name + "::STARTED"
        FINISHED = name + "::FINISHED"
        
        include Util::Observable
        
        # Creates a new TestRunnerMediator initialized to run
        # the passed suite.
        def initialize(suite)
          @suite = suite
        end

        # Runs the suite the TestRunnerMediator was created
        # with.
        def run_suite
          Unit.run = true
          begin_time = Time.now
          notify_listeners(RESET, @suite.size)
          result = create_result
          notify_listeners(STARTED, result)
          result_listener = result.add_listener(TestResult::CHANGED) do |updated_result|
            notify_listeners(TestResult::CHANGED, updated_result)
          end
          
          fault_listener = result.add_listener(TestResult::FAULT) do |fault|
            pp fault
            notify_listeners(TestResult::FAULT, fault)
          end
          
          @suite.run(result) do |channel, value|
            notify_listeners(channel, value)
          end
          
          result.remove_listener(TestResult::FAULT, fault_listener)
          result.remove_listener(TestResult::CHANGED, result_listener)
          end_time = Time.now
          elapsed_time = end_time - begin_time
          notify_listeners(FINISHED, elapsed_time) #"Finished in #{elapsed_time} seconds.")
          return result
        end

        private
        # A factory method to create the result the mediator
        # should run with. Can be overridden by subclasses if
        # one wants to use a different result.
        def create_result
          return TestResult.new
        end
      end

      class ThinTestRunner
        extend Test::Unit::UI::TestRunnerUtilities

        attr_reader :suites

        # Creates a new TestRunner for running the passed
        # suite. If quiet_mode is true, the output while
        # running is limited to progress dots, errors and
        # failures, and the final result. io specifies
        # where runner output should go to; defaults to
        # STDOUT.
        def initialize(suite, output_dir, output_level=NORMAL, io=STDOUT)
          @suite = suite
          @output_dir = output_dir
          @output_level = output_level
          @io = io
          @already_outputted = false
          @faults = []

          @total_passed = 0
          @total_failed = 0
          @suites = Hash.new {|hash, key| hash[key] = Array.new}
          @current_suite = nil
        end

        # Begins the test run.
        def start
          setup_mediator
          attach_to_mediator
          start_mediator
        end

        def get_binding
          binding
        end

        def total_failed
          total_tests - total_passed
        end

        private
        def setup_mediator
          @mediator = create_mediator(@suite)
          suite_name = @suite.to_s
          if @suite.kind_of?(Module)
            suite_name = @suite.name
          end
          output("Loaded suite #{suite_name}")
        end

        def create_mediator(suite)
          return DMTestMediator.new(suite)
        end

        def attach_to_mediator
          @mediator.add_listener(TestResult::FAULT, &method(:add_fault))
          @mediator.add_listener(TestRunnerMediator::STARTED, &method(:started))
          @mediator.add_listener(TestRunnerMediator::FINISHED, &method(:finished))
          @mediator.add_listener(TestCase::STARTED, &method(:test_started))
          @mediator.add_listener(TestCase::FINISHED, &method(:test_finished))
        end

        def start_mediator
          @mediator.run_suite
        end

        def add_fault(fault)
          error(fault.long_display)
          @current_test.add_fault(fault)

          @faults << fault
          output_single("FAIL".red, PROGRESS_ONLY)
          @already_outputted = true
        end

        def started(result)
          @result = result
          output("Started")
        end

        def finished(elapsed_time)
          nl
          output("Finished in #{elapsed_time} seconds.")
          @faults.each_with_index do |fault, index|
            nl
            output("%3d) %s" % [index + 1, fault.long_display])
          end
          nl
          output(@result)
        end

        def decompose_name(name)
          m = name.match(/test_(.*)[(](.*)[)]/)
          if m
            [m[2], m[1]]
          else
            ['anonymous', name]
          end
        end

        def result_file(name)
          "#{@output_dir}/#{mangle(name)}.result"
        end

        def log_file(name)
          "#{@output_dir}/#{mangle(name)}.result"
        end

        def yaml_file(name)
          s, n = decompose_name(name)
          "#{@output_dir}/#{mangle(s)}_#{mangle(n)}.yaml"
        end

        def test_started(name)
          suite, n = decompose_name(name)

          if @current_suite.nil? || @current_suite != suite
            output(suite.green, VERBOSE)
            @current_suite = suite
          end

          t = TestOutcome.new(suite, n, @output_dir)
          @current_log = File.open(t.log_path, 'w')
          set_log(@current_log)
          @current_test = t
          @suites[suite] << t
          output_single("  #{n.yellow}...", VERBOSE)
        end

        def test_finished(name)
          output_single("PASS".green, PROGRESS_ONLY) unless @already_outputted
          nl(VERBOSE)
          @already_outputted = false

          set_log(STDERR)
          @current_log.close
          File.open(yaml_file(name), 'w') do |file|
            file.puts @current_test.to_yaml
          end
        end

        def total_tests
          sum = 0
          @suites.values.each {|s| sum += s.size}
          sum
        end

        def total_passed
          sum = 0
          @suites.values.each do |s|
            s.each do |t|
              sum = sum + 1 if t.pass?
            end
          end
          sum
        end

        def nl(level=NORMAL)
          output("", level)
        end

        def output(something, level=NORMAL)
          @io.puts(something) if output?(level)
          @io.flush
        end

        def output_single(something, level=NORMAL)
          @io.write(something) if output?(level)
          @io.flush
        end

        def output?(level)
          level <= @output_level
        end
      end
    end
  end
end

#----------------------------------------------------------------

class Dispatcher
  def initialize
    @outcome_cache = OutcomeCache.new(log_dir)
  end

  def testcase_filter(filter)
    lambda {|t| filter.call(t.class.name.to_s)}
  end

  def name_filter(filter)
    lambda do |t|
      filter.call(trim_prefix('test_', t.method_name.to_s))
    end
  end

  def filter_combine_and(filters)
    lambda do |t|
      filters.each do |f|
        return false unless f.call(t)
      end

      true
    end
  end

  def filter_combine_or(filters)
    lambda do |t|
      filters.each do |f|
        return true if f.call(t)
      end

      false
    end
  end

  def help(opts, args)
    if args.size == 0
      usage
    else
      STDERR.puts "help not implemented yet"
    end
  end

  def setup_config(opts)
    setup_profile(opts.fetch(:profile, nil))
    setup_test_scale(opts.fetch(:test_scale, nil))
  end

  def run(opts, args)
    setup_config(opts)

    suite = select_tests(opts)
    runner = Test::Unit::UI::ThinTestRunner.new(suite, log_dir, Test::Unit::UI::VERBOSE)
    runner.start

    generate({}, [])

    if runner.total_failed > 0
      $failed = true
    end
  end

  def list(opts, args)
    setup_config(opts)

    suite = select_tests(opts)
    print_suite('', suite, opts.member?(:tags))
  end

  def serve(opts, args)
    config = {
      :Port => opts.fetch(:port, 8080),
      :DocumentRoot => report_dir
    }

    server = WEBrick::HTTPServer.new(config)

    ['INT', 'TERM'].each do |signal|
      trap(signal){ server.shutdown}
    end

    server.start
  end
  
  def self.load_file filename
    File.open(filename, 'r:bom|utf-8') {|f|
      input = f.read
      Psych.safe_load(input, :permitted_classes => [TestOutcome, Time])
    }
  end

  def generate(opts, args)
    generator = DMTest::ReportGenerator.new(report_dir)
    all_tests = Array.new
    
    Dir::glob("#{log_dir}/*.yaml") do |yaml_file|
      "found yaml file: #{yaml_file}"
      t = Dispatcher::load_file(yaml_file)
      if t.respond_to?(:suite)
        generator.unit_detail(t)
        all_tests << t
      end
    end

    generator.unit_summary(all_tests)
    generator.stylesheet
  end

  def global_command(opts, args)
    if args.size > 0
      STDERR.puts "Unknown command: #{args[0]}"
    else
      usage
    end
  end

  private
  def usage
    STDERR.puts <<EOF
Usage: dmtest <cmd> <opts>

  Where <cmd> is one of:

    help        - display help for a specific command
    generate    - regenerate reports (you rarely need to run this)
    list        - list tests
    run         - run tests
    serve       - serve the reports on a little web server
EOF
  end

  def setup_profile(sym = nil)
    cfg = get_config
    sym ||= cfg.default_profile
    $profile = cfg.profiles[sym]
    raise "unkown profile '#{sym}'" unless $profile
  end

  def setup_test_scale(sym = nil)
    cfg = get_config
    sym ||= cfg.default_test_scale
    $test_scale = sym
  end

  def get_config
    txt = File.read(config_file)
    c = DMTest::Config.new do
      eval(txt)
    end

    c
  end

  def select_tests(opts)
    filters = []
    if opts.member? :testcase
      filters << filter_combine_or(opts[:testcase].map {|p| testcase_filter(p)})
    end

    if opts.member? :name
      filters << filter_combine_or(opts[:name].map {|p| name_filter(p)})
    end

    suite = opts.fetch(:suite, nil)
    if suite.nil?
      # FIXME: usage
      STDERR.puts "please specify a test suite, eg, '--suite thin-provisioning'"
      exit(1)
    end

    require "dmtest/suites/#{suite}"

    if filters.size > 0
      filters = filter_combine_and(filters)
    end

    c = Test::Unit::Collector::ObjectSpace.new
    c.filter = filters
    c.collect(opts[:suite])
  end

  def trim_prefix(prefix, str)
    str.gsub(/^#{prefix}/, '')
  end

  def format_tags(tags)
    if tags.length == 0
      ''
    else
      str = tags.inject('') do |memo, tag|
        memo.length == 0 ? ":#{tag.to_s}" : "#{memo}, :#{tag.to_s}"
      end
    end
  end

  def print_suite(prefix, suite, want_tags)
    if suite.respond_to? :tests
      puts "#{prefix}#{suite}"
      prefix += '  '
      suite.tests.each {|t| print_suite(prefix, t, want_tags)}
    else
      tags = ''
      if want_tags && suite.class.respond_to?(:get_tags)
        tags = format_tags(suite.class.get_tags(suite.method_name)).blue
      end
      test = "#{prefix}#{trim_prefix('test_', suite.method_name.to_s)}"

      specifier = TestSpecifier.new(suite.class.to_s, trim_prefix('test_', suite.method_name.to_s))
      outcome = @outcome_cache.get_outcome(specifier)

      if outcome.nil?
        test = test.yellow
      elsif (outcome.pass?)
        test = test.green
      else
        test = test.red
      end

      puts "#{test}    #{tags}"
    end
  end

  #--------------------------------
  # dot dir stuff
  DOT_DIR = "#{ENV['HOME']}/.dmtest"

  def dot_dir
    unless File.directory?(DOT_DIR)
      setup_dot_dir
    end

    DOT_DIR
  end

  def report_dir_
    "#{DOT_DIR}/reports"
  end

  def log_dir_
    "#{DOT_DIR}/log"
  end

  def config_file_
    "#{DOT_DIR}/config"
  end

  def setup_dot_dir
    puts "creating #{DOT_DIR}"
    Dir.mkdir(DOT_DIR)

    puts "creating reports directory (#{report_dir_})"
    Dir.mkdir(report_dir_)

    puts "creating log directory (#{log_dir_})"
    Dir.mkdir(log_dir_)

    puts "writing example config file (#{config_file_}), please fill it in"
    File.open(config_file_, "w") do |f|
      f.write <<EOF
# profile :ssd do
#   metadata_dev '/dev/vdb'
#   data_dev '/dev/vdc'
# end
#
# profile :spindle do
#   metadata_dev '/dev/vdd'
#   data_dev '/dev/vde'
# end
#
# profile :mix do
#   metadata_dev '/dev/vdb'
#   data_dev '/dev/vde'
# end
#
# default_profile :ssd
# default_test_scale :normal
EOF
    end
  end

  def check_dot_dir
    unless File.directory? DOT_DIR
      setup_dot_dir
    end
  end

  def report_dir
    check_dot_dir
    report_dir_
  end

  def log_dir
    check_dot_dir
    log_dir_
  end

  def config_file
    check_dot_dir
    config_file_
  end
end

#----------------------------------------------------------------

def top_level_handler(&block)
  begin
    block.call
  rescue => e
    STDERR.puts e.message
    STDERR.puts e.backtrace
    exit 1
  end

  exit ($failed ? 1 : 0)
end

#----------------------------------------------------------------

top_level_handler do
  dispatcher = Dispatcher.new
  DMTestCommandLine.parse(dispatcher, *ARGV)
end

#----------------------------------------------------------------
#     o.on('-T', '--tags=TAG', String,
#          "Runs tests tagged with TAG (patterns may be used).") do |n|
#       n = (%r{\A/(.*)/\Z} =~ n ? Regexp.new($1) : n)
#       case n
#       when Regexp
#         $filters << lambda do |t|
#           begin
#             tags = t.class.get_tags(t.method_name.to_sym)
#             tags.any? do |tag|
#               n =~ tag.to_s
#             end ? true : nil
#           rescue
#             nil
#           end
#         end
#       else
#         $filters << lambda do |t|
#           begin
#             tags = t.class.get_tags(t.method_name.to_sym)
#             tags.any? do |tag|
#               n == tag.to_s
#             end ? true : nil
#           rescue
#             nil
#           end
#         end
#       end
#     end
#   end
# end