
from recipe_engine import recipe_api

class DartApi(recipe_api.RecipeApi):
  """Recipe module for code commonly used in dart recipes. Shouldn't be used elsewhere."""

  def checkout(self, channel=None, clobber=False):
    """Checks out the dart code and prepares it for building."""
    self.m.gclient.set_config('dart')
    # TODO(athom): Remove channel parameter from this function and fix URL
    # in gclient 'dart' config, remove line below.
    self.m.gclient.c.solutions[0].url = 'https://dart.googlesource.com/sdk.git'

    self.m.goma.ensure_goma()

    with self.m.context(cwd=self.m.path['cache'].join('builder'),
                        env={'GOMA_DIR':self.m.goma.goma_dir}):
      self.m.bot_update.ensure_checkout()
      with self.m.context(cwd=self.m.path['checkout']):
        if clobber:
          self.m.python('clobber',
                        self.m.path['checkout'].join('tools', 'clean_output_directory.py'))
      self.m.gclient.runhooks()

  def kill_tasks(self):
    """Kills leftover tasks from previous runs or steps."""
    self.m.python('kill processes',
               self.m.path['checkout'].join('tools', 'task_kill.py'),
               args=['--kill_browsers=True', '--kill_vsbuild=True'],
               ok_ret='any')

  def _swarming_checkout(self):
    try:
      if self.__swarming_checked_out:
        return
    except AttributeError:
      pass
    self.m.swarming_client.checkout(
      revision='5c8043e54541c3cee7ea255e0416020f2e3a5904')
    self.__swarming_checked_out = True

  def build(self, build_args=[], isolate=None, name='build dart'):
    """Builds dart using the specified build_args
       and optionally isolates the sdk for testing using the specified isolate.
       If an isolate is specified, it returns the hash of the isolated archive.
    """
    build_args = build_args + ['--no-start-goma']
    if self.m.platform.name in ('mac', 'win'):
      build_args.append('-j200')
    with self.m.context(cwd=self.m.path['checkout'],
                     env_prefixes={'PATH':[self.m.depot_tools.root]}):
      self.kill_tasks()
      try:
        self.m.goma.start()
        self.m.python(name,
                   self.m.path['checkout'].join('tools', 'build.py'),
                   args=build_args,
                   timeout=20 * 60)
      except self.m.step.StepTimeout as e:
        raise self.m.step.StepFailure('Step "%s" timed out after 20 minutes' % name)
      finally:
        self.m.goma.stop()

      if isolate is not None:
        self._swarming_checkout()
        bots_path = self.m.path['checkout'].join('tools', 'bots')
        isolate_paths = self.m.file.glob_paths("find isolate files", bots_path, '*.isolate',
                                          test_data=[bots_path.join('a.isolate'),
                                                     bots_path.join('b.isolate')])
        for path in isolate_paths:
          self.m.file.copy('copy %s to sdk root' % path.pieces[-1],
                           path,
                           self.m.path['checkout'])

        step_result = self.m.python(
          'upload testing isolate',
          self.m.swarming_client.path.join('isolate.py'),
          args= ['archive',
                 '--ignore_broken_items', # TODO(athom) find a way to avoid that
                 '-Ihttps://isolateserver.appspot.com',
                 '-i%s' % self.m.path['checkout'].join('%s.isolate' % isolate),
                 '-s%s' % self.m.path['checkout'].join('%s.isolated' % isolate)],
          stdout=self.m.raw_io.output('out'))
        isolate_hash = step_result.stdout.strip()[:40]
        step_result.presentation.step_text = 'isolate hash: %s' % isolate_hash
        return isolate_hash

  def upload_isolate(self, isolate_fileset):
    """Builds an isolate"""
    if isolate_fileset == self.m.properties.get('parent_fileset_name', None):
      return self.m.properties.get('parent_fileset')
    self._swarming_checkout()
    step_result = self.m.python(
        'upload testing fileset %s' % isolate_fileset,
        self.m.swarming_client.path.join('isolate.py'),
        args= ['archive',
                 '--ignore_broken_items', # TODO(athom) find a way to avoid that
                 '-Ihttps://isolateserver.appspot.com',
                 '-i%s' % self.m.path['checkout'].join('%s' % isolate_fileset),
                 '-s%s' % self.m.path['checkout'].join('%s.isolated' % isolate_fileset)],
        stdout=self.m.raw_io.output('out'))
    isolate_hash = step_result.stdout.strip()[:40]
    step_result.presentation.step_text = 'swarming fileset hash: %s' % isolate_hash
    return isolate_hash

  def download_parent_isolate(self):
    self._swarming_checkout()
    self.m.path['checkout'] = self.m.path['cleanup']
    isolate_hash = self.m.properties['parent_fileset']
    fileset_name = self.m.properties['parent_fileset_name']
    with self.m.context(cwd=self.m.path['cleanup']):
      step_result = self.m.python(
        'downloading fileset %s' % fileset_name,
        self.m.swarming_client.path.join('isolateserver.py'),
        args= ['download',
                 '-Ihttps://isolateserver.appspot.com',
                 '-s%s' % isolate_hash,
                 '--target=.'],
        stdout=self.m.raw_io.output('out'))

  def shard(self, title, isolate_hash, test_args, os=None, cpu='x86-64', pool='Dart.LUCI',
      num_shards=0, last_shard_is_local=False):
    """Runs test.py in the given isolate, sharded over several swarming tasks.
       Requires the 'shards' build property to be set to the number of tasks.
       Returns the created task(s), which are meant to be passed into collect().
    """
    if 'shards' in self.m.properties:
      num_shards = int(self.m.properties['shards'])
    assert(num_shards > 0)
    tasks = []
    for shard in range(num_shards - 1) if last_shard_is_local else range(num_shards):
      # TODO(athom) collect all the triggers, and present as a single step
      task = self.m.swarming.task("%s_shard_%s" % (title, (shard + 1)),
                               isolate_hash,
                               extra_args= test_args +
                                 ['--shards=%s' % num_shards,
                                  '--shard=%s' % (shard + 1),
                                  '--output_directory=${ISOLATED_OUTDIR}'])
      if os is None:
        os = self.m.platform.name
      os_names = {
        'win': 'Windows',
        'linux': 'Linux',
        'mac': 'Mac'
      }
      if os in os_names:
        os = os_names[os]
      task.dimensions['os'] = os
      # TODO(athom) remove this once all linux machines have chrome
      if os == 'Linux' and not '-rd8' in test_args:
        task.dimensions['kvm'] = '0'
      task.dimensions['cpu'] = cpu
      task.dimensions['pool'] = pool
      task.dimensions.pop('gpu', None)
      self.m.swarming.trigger_task(task)
      tasks.append(task)
    return tasks

  def collect(self, tasks):
    """Collects the results of a sharded test run."""
    # TODO(mkroghj) remove when all swarming recipes has been converted to neo.
    with self.m.step.defer_results():
      # TODO(athom) collect all the output, and present as a single step
      num_shards = int(self.m.properties['shards'])
      for shard in range(num_shards):
        task = tasks[shard]
        path = self.m.path['cleanup'].join(str(shard))
        task.task_output_dir = self.m.raw_io.output_dir(leak_to=path, name="results")
        collect = self.m.swarming.collect_task(task)
        output_dir = self.m.step.active_result.raw_io.output_dir
        for filename in output_dir:
          if "result.log" in filename: # pragma: no cover
            contents = output_dir[filename]
            self.m.step.active_result.presentation.logs['result.log'] = [contents]

  def collect_all(self, deferred_tasks):
    """Collects the results of a sharded test run."""
    with self.m.step.defer_results():
      # TODO(athom) collect all the output, and present as a single step
      for index_step,deferred_task in enumerate(deferred_tasks):
        if deferred_task.is_ok:
          for index_task,task in enumerate(deferred_task.get_result()):
            path = self.m.path['cleanup'].join(str(index_step) + '_' + str(index_task))
            task.task_output_dir = self.m.raw_io.output_dir(leak_to=path, name="results")
            collect = self.m.swarming.collect_task(task)
            output_dir = self.m.step.active_result.raw_io.output_dir
            for filename in output_dir:
              if "result.log" in filename: # pragma: no cover
                contents = output_dir[filename]
                self.m.step.active_result.presentation.logs['result.log'] = [contents]

  def read_result_file(self,  name, log_name, test_data=''):
    """Reads the result.log file
    Args:
      * name (str) - Name of step
      * log_name (str) - Name of log
      * test_data (str) - Some default data for this step to return when running
        under simulation.
    Returns (str) - The content of the file.
    Raises file.Error
    """
    result_log_path = self.m.path['checkout'].join('logs', 'result.log')
    try:
      read_data = self.m.file.read_text(
        name, result_log_path, test_data)
      self.m.step.active_result.presentation.logs[log_name] = [read_data]
      self.m.file.remove("delete result.log", result_log_path)
    except self.m.file.Error: # pragma: no cover
      pass

  def read_debug_log(self):
    """Reads the debug.log file"""
    if self.m.platform.name == 'win':
      self.m.step('debug log',
                  ['cmd.exe', '/c', 'type', '.debug.log'],
                  ok_ret='any')
    else:
      self.m.step('debug log',
                  ['cat', '.debug.log'],
                  ok_ret='any')

  def test(self, test_data):
    """Reads the test-matrix.json file in checkout and performs each step listed
    in the file

    Raises StepFailure.
    """
    test_matrix_path = self.m.path['checkout'].join('tools',
                                                    'bots',
                                                    'test_matrix.json')
    read_json = self.m.json.read(
      'read test-matrix.json',
      test_matrix_path,
      step_test_data=lambda: self.m.json.test_api.output(test_data))
    test_matrix = read_json.json.output
    builder = self.m.properties['buildername']
    if builder.endswith(('-be', '-try', '-stable', 'dev')):
      builder = builder[0:builder.rfind('-')]
    isolate_hashes = {}
    for config in test_matrix['configurations']:
      if builder in config['builders']:
        self._write_file_sets(test_matrix['filesets'])
        self._run_steps(config, isolate_hashes, builder)
        return
    raise self.m.step.StepFailure(
        'Error, could not find builder by name %s in test-matrix' % builder)

  def _write_file_sets(self, filesets):
    """Writes the fileset to the root of the sdk to allow for swarming to pick
    up the files and isolate the files.
    Args:
      * filesets - Filesets from the test-matrix
    """
    for fileset,files in filesets.iteritems():
      isolate_fileset = { 'variables': { 'files': files } }
      destination_path = self.m.path['checkout'].join(fileset)
      self.m.file.write_text('write fileset %s to sdk root' % fileset,
                            destination_path,
                            str(isolate_fileset))

  def _build_isolates(self, config, isolate_hashes):
    """Isolate filesets from all steps in config and returns a dictionary with a
    mapping from fileset to isolate_hash.
    Args:
      * config (dict) - Configuration of the builder, including the steps

    Returns (dict) - A mapping from fileset to isolate_hashes
    """
    for step in config['steps']:
      if 'fileset' in step and step['fileset'] not in isolate_hashes:
        isolate_hash = self.upload_isolate(step['fileset'])
        isolate_hashes[step['fileset']] = isolate_hash

  def _get_option(self, builder_fragments, options, default_value):
    """Gets an option from builder_fragments in options, or returns the default
    value."""
    intersection = set(builder_fragments) & set(options)
    if len(intersection) == 1:
      return intersection.pop()
    return default_value

  def _get_specific_argument(self, arguments, options):
    for arg in arguments:
      for option in options:
        if arg.startswith(option):
          return arg[len(option):]
    return None

  def _has_specific_argument(self, arguments, options):
    return self._get_specific_argument(arguments, options) is not None

  def _run_steps(self, config, isolate_hashes, builder_name):
    """Executes all steps from a json test-matrix builder entry"""
    # Find information from the builder name. It should be in the form
    # <info>-<os>-<mode>-<arch>-<runtime> or <info>-<os>-<mode>-<arch>.
    builder_fragments = builder_name.split('-')
    system = self._get_option(
      builder_fragments,
      ['linux', 'mac', 'win7', 'win8', 'win10'],
      'linux')
    mode = self._get_option(
      builder_fragments,
      ['debug', 'release', 'product'],
      'release')
    arch = self._get_option(
      builder_fragments,
      ['ia32', 'x64', 'arm', 'armv6', 'armv5te', 'arm64', 'simarm', 'simarmv6',
      'simarmv5te', 'simarm64', 'simdbc', 'simdbc64'],
      'x64')
    runtime = self._get_option(
      builder_fragments,
      ['none', 'd8', 'jsshell', 'ie9', 'ie10', 'ie11', 'ff',
            'safari', 'chrome', 'safarimobilesim', 'drt', 'ie10', 'ie11'],
      None)
    environment = {'system': system,
                   'mode': mode,
                   'arch': arch}
    if runtime is not None:
      environment['runtime'] = runtime
    channel = 'try'
    if 'branch' in self.m.properties:
      channels = {
        "refs/heads/master": "be",
        "refs/heads/stable": "stable",
        "refs/heads/dev": "dev"
      }
      channel = channels.get(self.m.properties['branch'], 'try');
    test_py_path = 'tools/test.py'
    build_py_path = 'tools/build.py'
    # Indexes the number of test.py steps.
    test_py_index = 0;
    tasks = []
    with self.m.step.defer_results():
      for index,step in enumerate(config['steps']):
        step_name = step['name']
        # If script is not defined, use test.py.
        script = step.get('script', test_py_path)
        args = step.get('arguments', [])
        is_build_step = script.endswith(build_py_path)
        is_trigger = 'trigger' in step
        is_test_py_step = script.endswith(test_py_path)
        script = self.m.path['checkout'].join(*script.split('/'))
        isolate_hash = None
        shards = step.get('shards', 0)
        local_shard = shards > 0 and index == len(config['steps']) - 1
        if 'fileset' in step:
          # We build isolates here, every time we see fileset, to wait for the
          # building of Dart, which may be included in the fileset.
          self._build_isolates(config, isolate_hashes)
          isolate_hash = isolate_hashes[step['fileset']]

        environment_variables = step.get('environment', {})
        environment_variables['BUILDBOT_BUILDERNAME'] = builder_name + "-%s" % channel
        with self.m.context(cwd=self.m.path['checkout'],
                            env=environment_variables,
                            env_prefixes={'PATH':[self.m.depot_tools.root]}):
          if is_build_step:
            if not self._has_specific_argument(args, ['-m', '--mode']):
              args = ['-m%s' % mode] + args
            if not self._has_specific_argument(args, ['-a', '--arch']):
              args = ['-a%s' % arch] + args
            self.build(name=step_name, build_args=args)
          elif is_trigger:
            self.run_trigger(step_name, step, isolate_hash)
          elif is_test_py_step:
            append_logs = test_py_index > 0
            self.run_test_py(step_name, append_logs, step,
                isolate_hash, shards, local_shard, environment, tasks)
            if shards == 0 or local_shard:
              # Only count indexes that are not sharded, to help with adding
              # append-logs.
              test_py_index += 1
          else:
            self.run_script(step_name, script, args, isolate_hash, shards,
                local_shard, environment, tasks)
      self.collect_all(tasks)

  def _copy_property(self, src, dest, key):
    if key in src:
      dest[key] = src[key]

  def run_trigger(self, step_name, step, isolate_hash):
    trigger_props = {}
    self._copy_property(self.m.properties, trigger_props, 'git_revision')
    self._copy_property(self.m.properties, trigger_props, 'revision')
    trigger_props['parent_buildername'] = self.m.properties['buildername']
    trigger_props['parent_build_id'] = self.m.properties.get('build_id', '')
    if isolate_hash:
      trigger_props['parent_fileset'] = isolate_hash
      trigger_props['parent_fileset_name'] = step['fileset']
    put_result = self.m.buildbucket.put(
        [
          {
            'bucket': 'luci.dart.ci',
            'parameters': {
              'builder_name': builder_name,
              'properties': trigger_props,
              'changes': [
                {
                  'author': {
                    'email': author,
                  },
                }
                for author in self.m.properties.get('blamelist', [])
              ],
            },
          }
          for builder_name in step['trigger']
        ])
    self.m.step.active_result.presentation.step_text = step_name
    for build in put_result.stdout['results']:
      builder_tag = (x for x in build['build']['tags'] if x.startswith('builder:')).next()
      builder_name = builder_tag[len('builder:'):]
      self.m.step.active_result.presentation.links[builder_name] = build['build']['url']

  def run_test_py(self, step_name, append_logs, step, isolate_hash, shards,
      local_shard, environment, tasks):
    """Runs test.py with default arguments, based on configuration from.
    Args:
      * step_name (str) - Name of the step
      * append_logs (bool) - Add append_log to arguments
      * step (dict) - Test-matrix step
      * isolate_hash (String) - Hash of uploadet fileset/isolate if the
        process is to be sharded
      * shards (int) - The number of shards
      * local_shard (bool) - Should the current builder be one of the shards.
      * environment (dict) - Environment with runtime, arch, system etc
      * tasks ([task]) - placeholder to put all swarming tasks in
    """
    args = step.get('arguments', [])
    test_args = ['--progress=buildbot',
                 '-v',
                 '--report',
                 '--time',
                 '--write-debug-log',
                 '--write-result-log',
                 '--write-test-outcome-log']
    if not self._has_specific_argument(args, ['-m', '--mode']):
      test_args = ['-m%s' % environment['mode']] + test_args
    if not self._has_specific_argument(args, ['-a', '--arch']):
      test_args = ['-a%s' % environment['arch']] + test_args
    if 'runtime' in environment and not self._has_specific_argument(
        args, ['-r', '--runtime']):
      test_args = test_args + ['-r%s' % environment['runtime']]
    args = test_args + args
    if append_logs:
      args = args + ['--append_logs']
    if environment['system'] in ['win7', 'win8', 'win10']:
      args = args + ['--builder-tag=%s' % environment['system']]
    if 'exclude_tests' in step:
        args = args + ['--exclude_suite=' + ','.join(step['exclude_tests'])]
    if 'tests' in step:
      args = args + step['tests']
    self.run_script(step_name, 'tools/test.py', args, isolate_hash, shards,
        local_shard, environment, tasks)
    if shards == 0 or local_shard:
      self.read_result_file('read results of %s' % step_name, 'result.log')

  def run_script(self, step_name, script, args, isolate_hash, shards,
      local_shard, environment, tasks):
    """Runs a specific script with current working directory to be checkout. If
    the runtime (passed in environment) is a browser, and the system is linux,
    xvfb is used. If an isolate_hash is passed in, it will swarm the command.
    Args:
      * step_name (str) - Name of the step
      * script (str) - The script to invoke
      * args ([str]) - Additional arguments to test.py
      * isolate_hash (str) - The isolate hash if the script should be swarmed
      * shards (int) - The number of shards to invoke
      * local_shard (bool) - Should the current builder be used as a shard
      * environment (dict) - Environment with runtime, arch, system etc
      * tasks ([task]) - placeholder to put all swarming tasks in
    """
    runtime = self._get_specific_argument(args, ['-r', '--runtime'])
    if runtime is None:
      runtime = environment.get('runtime', None)
    use_xvfb = (runtime in ['drt', 'chrome', 'ff'] and
                environment['system'] == 'linux')
    with self.m.step.defer_results():
      if use_xvfb:
        xvfb_cmd = [
          '/usr/bin/xvfb-run',
          '-a',
          '--server-args=-screen 0 1024x768x24']
        cmd = xvfb_cmd + ['python', '-u', script] + args
        if isolate_hash:
          tasks.append(self.shard(step_name, isolate_hash, cmd,
              num_shards=shards, last_shard_is_local=local_shard))
        else:
          self.m.step(step_name, cmd)
      else:
        if isolate_hash:
          tasks.append(self.shard(step_name, isolate_hash, [script] + args,
              num_shards=shards, last_shard_is_local=local_shard))
        elif '.py' in str(script):
          self.m.python(step_name, script, args=args)
        else:
          self.m.step(step_name, [script] + args)

    if local_shard:
      this_shard = shards + 1
      args = args + [
        '--shards=%s' % this_shard,
        '--shard=%s' % this_shard
      ]
      self.run_script("%s_shard_%s" % (step_name, this_shard), script,
          args, None, 0, False, environment, tasks)