/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.hive.hcatalog.hbase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;

import java.io.File;
import java.io.IOException;
import java.net.ServerSocket;

/**
 * MiniCluster class composed of a number of Hadoop Minicluster implementations
 * and other necessary daemons needed for testing (HBase, Hive MetaStore, Zookeeper, MiniMRCluster)
 */
public class ManyMiniCluster {

  //MR stuff
  private boolean miniMRClusterEnabled;
  private MiniMRCluster mrCluster;
  private int numTaskTrackers;
  private JobConf jobConf;

  //HBase stuff
  private boolean miniHBaseClusterEnabled;
  private MiniHBaseCluster hbaseCluster;
  private String hbaseRoot;
  private Configuration hbaseConf;
  private String hbaseDir;

  //ZK Stuff
  private boolean miniZookeeperClusterEnabled;
  private MiniZooKeeperCluster zookeeperCluster;
  private int zookeeperPort;
  private String zookeeperDir;

  //DFS Stuff
  private MiniDFSCluster dfsCluster;

  //Hive Stuff
  private boolean miniHiveMetastoreEnabled;
  private HiveConf hiveConf;
  private HiveMetaStoreClient hiveMetaStoreClient;

  private final File workDir;
  private boolean started = false;


  /**
   * create a cluster instance using a builder which will expose configurable options
   * @param workDir working directory ManyMiniCluster will use for all of it's *Minicluster instances
   * @return a Builder instance
   */
  public static Builder create(File workDir) {
    return new Builder(workDir);
  }

  private ManyMiniCluster(Builder b) {
    workDir = b.workDir;
    numTaskTrackers = b.numTaskTrackers;
    hiveConf = b.hiveConf;
    jobConf = b.jobConf;
    hbaseConf = b.hbaseConf;
    miniMRClusterEnabled = b.miniMRClusterEnabled;
    miniHBaseClusterEnabled = b.miniHBaseClusterEnabled;
    miniHiveMetastoreEnabled = b.miniHiveMetastoreEnabled;
    miniZookeeperClusterEnabled = b.miniZookeeperClusterEnabled;
  }

  protected synchronized void start() {
    try {
      if (!started) {
        FileUtil.fullyDelete(workDir);
        if (miniMRClusterEnabled) {
          setupMRCluster();
        }
        if (miniZookeeperClusterEnabled || miniHBaseClusterEnabled) {
          miniZookeeperClusterEnabled = true;
          setupZookeeper();
        }
        if (miniHBaseClusterEnabled) {
          setupHBaseCluster();
        }
        if (miniHiveMetastoreEnabled) {
          setUpMetastore();
        }
      }
    } catch (Exception e) {
      throw new IllegalStateException("Failed to setup cluster", e);
    }
  }

  protected synchronized void stop() {
    if (hbaseCluster != null) {
      try {
        hbaseCluster.shutdown();
      } catch (Exception e) {
        e.printStackTrace();
      }
      hbaseCluster = null;
    }
    if (zookeeperCluster != null) {
      try {
        zookeeperCluster.shutdown();
      } catch (Exception e) {
        e.printStackTrace();
      }
      zookeeperCluster = null;
    }
    if (mrCluster != null) {
      try {
        mrCluster.shutdown();
      } catch (Exception e) {
        e.printStackTrace();
      }
      mrCluster = null;
    }
    if (dfsCluster != null) {
      try {
        dfsCluster.getFileSystem().close();
        dfsCluster.shutdown();
      } catch (Exception e) {
        e.printStackTrace();
      }
      dfsCluster = null;
    }
    try {
      FileSystem.closeAll();
    } catch (IOException e) {
      e.printStackTrace();
    }
    started = false;
  }

  /**
   * @return Configuration of mini HBase cluster
   */
  public Configuration getHBaseConf() {
    return HBaseConfiguration.create(hbaseConf);
  }

  /**
   * @return Configuration of mini MR cluster
   */
  public Configuration getJobConf() {
    return new Configuration(jobConf);
  }

  /**
   * @return Configuration of Hive Metastore, this is a standalone not a daemon
   */
  public HiveConf getHiveConf() {
    return new HiveConf(hiveConf);
  }

  /**
   * @return Filesystem used by MiniMRCluster and MiniHBaseCluster
   */
  public FileSystem getFileSystem() {
    try {
      return FileSystem.get(jobConf);
    } catch (IOException e) {
      throw new IllegalStateException("Failed to get FileSystem", e);
    }
  }

  /**
   * @return Metastore client instance
   */
  public HiveMetaStoreClient getHiveMetaStoreClient() {
    return hiveMetaStoreClient;
  }

  private void setupMRCluster() {
    try {
      final int jobTrackerPort = findFreePort();
      final int taskTrackerPort = findFreePort();

      if (jobConf == null)
        jobConf = new JobConf();

      jobConf.setInt("mapred.submit.replication", 1);
      jobConf.set("yarn.scheduler.capacity.root.queues", "default");
      jobConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
      //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history");
      System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());

      mrCluster = new MiniMRCluster(jobTrackerPort,
        taskTrackerPort,
        numTaskTrackers,
        getFileSystem().getUri().toString(),
        numTaskTrackers,
        null,
        null,
        null,
        jobConf);

      jobConf = mrCluster.createJobConf();
    } catch (IOException e) {
      throw new IllegalStateException("Failed to Setup MR Cluster", e);
    }
  }

  private void setupZookeeper() {
    try {
      zookeeperDir = new File(workDir, "zk").getAbsolutePath();
      zookeeperPort = findFreePort();
      zookeeperCluster = new MiniZooKeeperCluster();
      zookeeperCluster.setDefaultClientPort(zookeeperPort);
      zookeeperCluster.startup(new File(zookeeperDir));
    } catch (Exception e) {
      throw new IllegalStateException("Failed to Setup Zookeeper Cluster", e);
    }
  }

  private void setupHBaseCluster() {
    final int numRegionServers = 1;
    Connection connection = null;
    Table table = null;

    try {
      hbaseDir = new File(workDir, "hbase").getCanonicalPath();
      hbaseDir = hbaseDir.replaceAll("\\\\", "/");
      hbaseRoot = "file:///" + hbaseDir;

      if (hbaseConf == null)
        hbaseConf = HBaseConfiguration.create();

      hbaseConf.set("hbase.rootdir", hbaseRoot);
      hbaseConf.set("hbase.master", "local");
      hbaseConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zookeeperPort);
      hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
      hbaseConf.setInt("hbase.master.port", findFreePort());
      hbaseConf.setInt("hbase.master.info.port", -1);
      hbaseConf.setInt("hbase.regionserver.port", findFreePort());
      hbaseConf.setInt("hbase.regionserver.info.port", -1);

      hbaseCluster = new MiniHBaseCluster(hbaseConf, numRegionServers);
      hbaseConf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
      //opening the META table ensures that cluster is running
      connection = ConnectionFactory.createConnection(hbaseConf);
      table = connection.getTable(TableName.META_TABLE_NAME);
    } catch (Exception e) {
      throw new IllegalStateException("Failed to setup HBase Cluster", e);
    } finally {
      if (table != null) {
        try {
          table.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (connection != null) {
        try {
          connection.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    }
  }

  private void setUpMetastore() throws Exception {
    if (hiveConf == null)
      hiveConf = new HiveConf(this.getClass());

    //The default org.apache.hadoop.hive.ql.hooks.PreExecutePrinter hook
    //is present only in the ql/test directory
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
      "jdbc:derby:" + new File(workDir + "/metastore_db") + ";create=true");
    hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.toString(),
      new File(workDir, "warehouse").toString());
    //set where derby logs
    File derbyLogFile = new File(workDir + "/derby.log");
    derbyLogFile.createNewFile();
    System.setProperty("derby.stream.error.file", derbyLogFile.getPath());


//    Driver driver = new Driver(conf);
//    SessionState.start(new CliSessionState(conf));

    hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf);
  }

  private static int findFreePort() throws IOException {
    ServerSocket server = new ServerSocket(0);
    int port = server.getLocalPort();
    server.close();
    return port;
  }

  public static class Builder {
    private File workDir;
    private int numTaskTrackers = 1;
    private JobConf jobConf;
    private Configuration hbaseConf;
    private HiveConf hiveConf;

    private boolean miniMRClusterEnabled = true;
    private boolean miniHBaseClusterEnabled = true;
    private boolean miniHiveMetastoreEnabled = true;
    private boolean miniZookeeperClusterEnabled = true;


    private Builder(File workDir) {
      this.workDir = workDir;
    }

    public Builder numTaskTrackers(int num) {
      numTaskTrackers = num;
      return this;
    }

    public Builder jobConf(JobConf jobConf) {
      this.jobConf = jobConf;
      return this;
    }

    public Builder hbaseConf(Configuration hbaseConf) {
      this.hbaseConf = hbaseConf;
      return this;
    }

    public Builder hiveConf(HiveConf hiveConf) {
      this.hiveConf = hiveConf;
      return this;
    }

    public Builder miniMRClusterEnabled(boolean enabled) {
      this.miniMRClusterEnabled = enabled;
      return this;
    }

    public Builder miniHBaseClusterEnabled(boolean enabled) {
      this.miniHBaseClusterEnabled = enabled;
      return this;
    }

    public Builder miniZookeeperClusterEnabled(boolean enabled) {
      this.miniZookeeperClusterEnabled = enabled;
      return this;
    }

    public Builder miniHiveMetastoreEnabled(boolean enabled) {
      this.miniHiveMetastoreEnabled = enabled;
      return this;
    }


    public ManyMiniCluster build() {
      return new ManyMiniCluster(this);
    }

  }
}
