/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.ptest.execution;

import java.io.File;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
import org.apache.hive.ptest.execution.conf.Context;
import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
import org.apache.hive.ptest.execution.conf.Host;
import org.apache.hive.ptest.execution.conf.TestConfiguration;
import org.apache.hive.ptest.execution.conf.TestParser;
import org.apache.hive.ptest.execution.context.ExecutionContext;
import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
import org.apache.velocity.app.Velocity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;

public class PTest {

  static {
    Velocity.init();
  }
  private static final Logger LOG = LoggerFactory
      .getLogger(PTest.class);


  private final TestConfiguration mConfiguration;
  private final ListeningExecutorService mExecutor;
  private final Set<String> mAddedTests;
  private final Set<String> mExecutedTests;
  private final Set<String> mFailedTests;
  private final List<Phase> mPhases;
  private final ExecutionContext mExecutionContext;
  private final Logger mLogger;
  private final List<HostExecutor> mHostExecutors;
  private final String mBuildTag;
  private final SSHCommandExecutor mSshCommandExecutor;
  private final RSyncCommandExecutor mRsyncCommandExecutor;
  private final ImmutableMap<String, String> templateDefaults;

  public PTest(final TestConfiguration configuration, final ExecutionContext executionContext,
      final String buildTag, final File logDir, final LocalCommandFactory localCommandFactory,
      final SSHCommandExecutor sshCommandExecutor, final  RSyncCommandExecutor rsyncCommandExecutor,
      final Logger logger) throws Exception {
    mConfiguration = configuration;
    mLogger = logger;
    mBuildTag = buildTag;
    mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
    mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
    mAddedTests = new HashSet<String>();
    mExecutionContext = executionContext;
    mSshCommandExecutor = sshCommandExecutor;
    mRsyncCommandExecutor = rsyncCommandExecutor;
    mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool(
        new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HostExecutor %d").build()));
    final File failedLogDir = Dirs.create(new File(logDir, "failed"));
    final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
    final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
    File patchDir = Dirs.createEmpty(new File(logDir, "patches"));
    File patchFile = null;
    if(!configuration.getPatch().isEmpty()) {
      patchFile = new File(patchDir, buildTag + ".patch");
      Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile);
    }
    ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder();
    templateDefaultsBuilder.
    put("repository", configuration.getRepository()).
    put("repositoryName", configuration.getRepositoryName()).
    put("repositoryType", configuration.getRepositoryType()).
    put("buildTool", configuration.getBuildTool()).
    put("branch", configuration.getBranch()).
    put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache())).
    put("workingDir", mExecutionContext.getLocalWorkingDirectory()).
    put("buildTag", buildTag).
    put("logDir", logDir.getAbsolutePath()).
    put("javaHome", configuration.getJavaHome()).
    put("javaHomeForTests", configuration.getJavaHomeForTests()).
    put("antEnvOpts", configuration.getAntEnvOpts()).
    put("antArgs", configuration.getAntArgs()).
    put("antTestArgs", configuration.getAntTestArgs()).
    put("antTestTarget", configuration.getAntTestTarget()).
    put("mavenEnvOpts", configuration.getMavenEnvOpts()).
    put("mavenArgs", configuration.getMavenArgs()).
    put("mavenBuildArgs", configuration.getMavenBuildArgs()).
    put("mavenTestArgs", configuration.getMavenTestArgs());
    if (!Strings.isNullOrEmpty(configuration.getAdditionalProfiles())) {
      templateDefaultsBuilder.put("additionalProfiles", configuration.getAdditionalProfiles());
    }
    templateDefaults = templateDefaultsBuilder.build();
    TestParser testParser = new TestParser(configuration.getContext(), new AtomicInteger(1), configuration.getTestCasePropertyName(),
        new File(mExecutionContext.getLocalWorkingDirectory(), configuration.getRepositoryName() + "-source"),
        logger);

    HostExecutorBuilder hostExecutorBuilder = new HostExecutorBuilder() {
      @Override
      public HostExecutor build(Host host) {
        return new HostExecutor(host, executionContext.getPrivateKey(), mExecutor, sshCommandExecutor,
            rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10,
            configuration.shouldFetchLogsForSuccessfulTests(), logger);
      }

    };
    List<HostExecutor> hostExecutors = new ArrayList<HostExecutor>();
    for(Host host : mExecutionContext.getHosts()) {
      hostExecutors.add(hostExecutorBuilder.build(host));
    }
    mHostExecutors = new CopyOnWriteArrayList<HostExecutor>(hostExecutors);
    mPhases = Lists.newArrayList();
    mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, patchFile, logger, mAddedTests));
    mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
    mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults,
        mExecutionContext.getLocalWorkingDirectory(), scratchDir, logger, logDir, patchFile));
    mPhases.add(new ExecutionPhase(mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults,
        succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger));
    mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
  }
  public int run() {
    int result = 0;
    boolean error = false;
    List<String> messages = Lists.newArrayList();
    Map<String, Long> elapsedTimes = Maps.newTreeMap();
    try {
      mLogger.info("Running tests with " + mConfiguration);
      mLogger.info("Running tests with configuration context=[{}]", mConfiguration.getContext());
      for(Phase phase : mPhases) {
        String msg = "Executing " + phase.getClass().getName();
        mLogger.info(msg);
        messages.add(msg);
        long start = System.currentTimeMillis();
        try {
          phase.execute();
        } finally {
          long elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
              TimeUnit.MILLISECONDS);
          elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime);
        }
      }
      if(!mFailedTests.isEmpty()) {
        throw new TestsFailedException(mFailedTests.size() + " tests failed");
      }
    } catch(Throwable throwable) {
      mLogger.error("Test run exited with an unexpected error", throwable);
      // NonZeroExitCodeExceptions can have long messages and should be
      // trimmable when published to the JIRA via the JiraService
      if(throwable instanceof NonZeroExitCodeException) {
        messages.add("Tests exited with: " + throwable.getClass().getSimpleName());
        for(String line : Strings.nullToEmpty(throwable.getMessage()).split("\n")) {
          messages.add(line);
        }
      } else {
        messages.add("Tests exited with: " + throwable.getClass().getSimpleName() +
            ": " + throwable.getMessage());
      }
      error = true;
    } finally {
      for(HostExecutor hostExecutor : mHostExecutors) {
        hostExecutor.shutdownNow();
        if(hostExecutor.isBad()) {
          mExecutionContext.addBadHost(hostExecutor.getHost());
        }
      }
      mSshCommandExecutor.shutdownNow();
      mRsyncCommandExecutor.shutdownNow();
      mExecutor.shutdownNow();
      SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
      if(failedTests.isEmpty()) {
        mLogger.info(String.format("%d failed tests", failedTests.size()));
      } else {
        mLogger.warn(String.format("%d failed tests", failedTests.size()));
      }
      for(String failingTestName : failedTests) {
        mLogger.warn(failingTestName);
      }
      mLogger.info("Executed " + mExecutedTests.size() + " tests");
      for(Map.Entry<String, Long> entry : elapsedTimes.entrySet()) {
        mLogger.info(String.format("PERF: Phase %s took %d minutes", entry.getKey(), entry.getValue()));
      }
      publishJiraComment(error, messages, failedTests, mAddedTests);
      if(error || !mFailedTests.isEmpty()) {
        result = 1;
      }
    }
    return result;
  }

  private void publishJiraComment(boolean error, List<String> messages, SortedSet<String> failedTests, Set<String> addedTests) {
    if(mConfiguration.getJiraName().isEmpty()) {
      mLogger.info("Skipping JIRA comment as name is empty.");
      return;
    }
    if(mConfiguration.getJiraUrl().isEmpty()) {
      mLogger.info("Skipping JIRA comment as URL is empty.");
      return;
    }
    if(mConfiguration.getJiraUser().isEmpty()) {
      mLogger.info("Skipping JIRA comment as user is empty.");
      return;
    }
    if(mConfiguration.getJiraPassword().isEmpty()) {
      mLogger.info("Skipping JIRA comment as password is empty.");
      return;
    }
    mLogger.info("Added tests: " + addedTests);
    JIRAService jira = new JIRAService(mLogger, mConfiguration, mBuildTag);
    jira.postComment(error, mExecutedTests.size(), failedTests, messages, addedTests);
  }

  public static class Builder {
    public PTest build(TestConfiguration configuration, ExecutionContext executionContext,
        String buildTag, File logDir, LocalCommandFactory localCommandFactory, SSHCommandExecutor sshCommandExecutor,
        RSyncCommandExecutor rsyncCommandExecutor, Logger logger) throws Exception {
      return new PTest(configuration, executionContext, buildTag, logDir, localCommandFactory, sshCommandExecutor,
          rsyncCommandExecutor, logger);
    }
  }

  private static final String PROPERTIES = "properties";
  private static final String SERVER_ENV_PROPERTIES = "hive.ptest.server.env.properties";
  private static final String REPOSITORY = TestConfiguration.REPOSITORY;
  private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME;
  private static final String BRANCH = TestConfiguration.BRANCH;
  private static final String PATCH = "patch";
  private static final String JAVA_HOME = TestConfiguration.JAVA_HOME;
  private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST;
  private static final String ANT_TEST_ARGS = TestConfiguration.ANT_TEST_ARGS;
  private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS;
  private static final String ANT_TEST_TARGET = TestConfiguration.ANT_TEST_TARGET;
  /**
   * All args override properties file settings except
   * for this one which is additive.
   */
  private static final String ANT_ARG = "D";

  public static void main(String[] args) throws Exception {
    LOG.info("Args " + Arrays.toString(args));
    CommandLineParser parser = new GnuParser();
    Options options = new Options();
    options.addOption(null, PROPERTIES, true, "properties file");
    options.addOption(null, SERVER_ENV_PROPERTIES, true, "optional properties file with environment properties");
    options.addOption(null, REPOSITORY, true, "Overrides git repository in properties file");
    options.addOption(null, REPOSITORY_NAME, true, "Overrides git repository *name* in properties file");
    options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
    options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
    options.addOption(ANT_ARG, null, true, "Supplemental ant arguments");
    options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)");
    options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
    options.addOption(null, ANT_TEST_ARGS, true, "Arguments to ant test on slave nodes only");
    options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environment variable setting");
    CommandLine commandLine = parser.parse(options, args);
    if(!commandLine.hasOption(PROPERTIES)) {
      throw new IllegalArgumentException(Joiner.on(" ").
          join(PTest.class.getName(), "--" + PROPERTIES,"config.properties"));
    }
    String testConfigurationFile = commandLine.getOptionValue(PROPERTIES);
    String environmentConfigurationFile = commandLine.getOptionValue(SERVER_ENV_PROPERTIES);

    Context.ContextBuilder builder = new Context.ContextBuilder();
    builder.addPropertiesFile(testConfigurationFile);

    if (environmentConfigurationFile != null) {
      builder.addPropertiesFile(environmentConfigurationFile);
    }

    Context ctx = builder.build();

    ExecutionContextConfiguration executionContextConfiguration = ExecutionContextConfiguration.
        withContext(ctx);
    String buildTag = System.getenv("BUILD_TAG") == null ? "undefined-"
        + System.currentTimeMillis() : System.getenv("BUILD_TAG");
        File logDir = Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag));
    LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(executionContextConfiguration.
        getGlobalLogDirectory()), 5);
    cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory());
    cleaner.setDaemon(true);
    cleaner.start();
    TestConfiguration conf = TestConfiguration.withContext(ctx, LOG);
    String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim();
    if(!repository.isEmpty()) {
      conf.setRepository(repository);
    }
    String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim();
    if(!repositoryName.isEmpty()) {
      conf.setRepositoryName(repositoryName);
    }
    String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim();
    if(!branch.isEmpty()) {
      conf.setBranch(branch);
    }
    String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim();
    if(!patch.isEmpty()) {
      conf.setPatch(patch);
    }
    String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim();
    if(!javaHome.isEmpty()) {
      conf.setJavaHome(javaHome);
    }
    String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
    if(!javaHomeForTests.isEmpty()) {
      conf.setJavaHomeForTests(javaHomeForTests);
    }
    String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim();
    if(!antTestArgs.isEmpty()) {
      conf.setAntTestArgs(antTestArgs);
    }
    String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
    if(!antEnvOpts.isEmpty()) {
      conf.setAntEnvOpts(antEnvOpts);
    }
    String antTestTarget = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_TARGET)).trim();
    if(!antTestTarget.isEmpty()) {
      conf.setAntTestTarget(antTestTarget);
    }
    String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG);
    if(supplementalAntArgs != null && supplementalAntArgs.length > 0) {
      String antArgs = Strings.nullToEmpty(conf.getAntArgs());
      if(!(antArgs.isEmpty() || antArgs.endsWith(" "))) {
        antArgs += " ";
      }
      antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs);
      conf.setAntArgs(antArgs);
    }
    ExecutionContextProvider executionContextProvider = null;
    ExecutionContext executionContext = null;
    int exitCode = 0;
    try {
      executionContextProvider = executionContextConfiguration
          .getExecutionContextProvider();
      executionContext = executionContextProvider.createExecutionContext();
      LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG);
      PTest ptest = new PTest(conf, executionContext, buildTag, logDir,
          localCommandFactory, new SSHCommandExecutor(LOG, localCommandFactory, conf.getSshOpts()),
          new RSyncCommandExecutor(LOG, 10, localCommandFactory), LOG);
      exitCode = ptest.run();
    } finally {
      if(executionContext != null) {
        executionContext.terminate();
      }
      if(executionContextProvider != null) {
        executionContextProvider.close();
      }
    }
    System.exit(exitCode);
  }

  @VisibleForTesting
  public Map<String, String> getTemplateDefaults() {
    return templateDefaults;
  }
}
