/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hive.serde2.io;

import com.google.code.tempusfugit.concurrency.annotations.*;
import com.google.code.tempusfugit.concurrency.*;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.TimeZone;

import org.apache.hadoop.hive.common.type.TimestampUtils;
import org.junit.*;
import static org.junit.Assert.*;

import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;

public class TestTimestampWritableV2 {

  @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
  @Rule public RepeatingRule repeatingRule = new RepeatingRule();

  private static ThreadLocal<DateFormat> DATE_FORMAT =
      new ThreadLocal<DateFormat>() {
        @Override
        protected DateFormat initialValue() {
          SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
          formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
          return formatter;
        }
      };

  private static final int HAS_DECIMAL_MASK = 0x80000000;

  private static final long MAX_ADDITIONAL_SECONDS_BITS = 0x418937;

  private static long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("0001-01-01 00:00:00");
  private static long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("9999-01-01 00:00:00");

  private static int BILLION = 1000 * 1000 * 1000;

  private static long getSeconds(Timestamp ts) {
    // To compute seconds, we first subtract the milliseconds stored in the nanos field of the
    // Timestamp from the result of getTime().
    long seconds = (ts.toEpochMilli() - ts.getNanos() / 1000000) / 1000;

    // It should also be possible to calculate this based on ts.getTime() only.
    assertEquals(seconds, TimestampUtils.millisToSeconds(ts.toEpochMilli()));

    return seconds;
  }

  private static long parseToMillis(String s) {
    try {
      return DATE_FORMAT.get().parse(s).getTime();
    } catch (ParseException ex) {
      throw new RuntimeException(ex);
    }
  }

  @Before
  public void setUp() {
    TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
  }

  private static String normalizeTimestampStr(String timestampStr) {
    if (timestampStr.endsWith(".0")) {
      return timestampStr.substring(0, timestampStr.length() - 2);
    }
    return timestampStr;
  }

  private static void assertTSWEquals(TimestampWritableV2 expected, TimestampWritableV2 actual) {
    assertEquals(normalizeTimestampStr(expected.toString()),
                 normalizeTimestampStr(actual.toString()));
    assertEquals(expected, actual);
    assertEquals(expected.getTimestamp(), actual.getTimestamp());
  }

  private static TimestampWritableV2 deserializeFromBytes(byte[] tsBytes) throws IOException {
    ByteArrayInputStream bais = new ByteArrayInputStream(tsBytes);
    DataInputStream dis = new DataInputStream(bais);
    TimestampWritableV2 deserTSW = new TimestampWritableV2();
    deserTSW.readFields(dis);
    return deserTSW;
  }

  private static int reverseNanos(int nanos) {
    if (nanos == 0) {
      return 0;
    }
    if (nanos < 0 || nanos >= 1000 * 1000 * 1000) {
      throw new IllegalArgumentException("Invalid nanosecond value: " + nanos);
    }

    int x = nanos;
    StringBuilder reversed = new StringBuilder();
    while (x != 0) {
      reversed.append((char)('0' + x % 10));
      x /= 10;
    }

    int result = Integer.parseInt(reversed.toString());
    while (nanos < 100 * 1000 * 1000) {
      result *= 10;
      nanos *= 10;
    }
    return result;
  }

  private static byte[] serializeToBytes(Writable w) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    w.write(dos);
    return baos.toByteArray();
  }

  private static List<Byte> toList(byte[] a) {
    List<Byte> list = new ArrayList<Byte>(a.length);
    for (byte b : a) {
      list.add(b);
    }
    return list;
  }

  /**
   * Pad the given byte array with the given number of bytes in the beginning. The padding bytes
   * deterministically depend on the passed data.
   */
  private static byte[] padBytes(byte[] bytes, int count) {
    byte[] result = new byte[bytes.length + count];
    for (int i = 0; i < count; ++i) {
      // Fill the prefix bytes with deterministic data based on the actual meaningful data.
      result[i] = (byte) (bytes[i % bytes.length] * 37 + 19);
    }
    System.arraycopy(bytes, 0, result, count, bytes.length);
    return result;
  }

  private static TimestampWritableV2 serializeDeserializeAndCheckTimestamp(Timestamp ts)
      throws IOException {
    TimestampWritableV2 tsw = new TimestampWritableV2(ts);
    assertEquals(ts, tsw.getTimestamp());

    byte[] tsBytes = serializeToBytes(tsw);
    TimestampWritableV2 deserTSW = deserializeFromBytes(tsBytes);
    assertTSWEquals(tsw, deserTSW);
    assertEquals(ts, deserTSW.getTimestamp());
    assertEquals(tsBytes.length, tsw.getTotalLength());

    // Also convert to/from binary-sortable representation.
    int binarySortableOffset = Math.abs(tsw.hashCode()) % 10;
    byte[] binarySortableBytes = padBytes(tsw.getBinarySortable(), binarySortableOffset);
    TimestampWritableV2 fromBinSort = new TimestampWritableV2();
    fromBinSort.setBinarySortable(binarySortableBytes, binarySortableOffset);
    assertTSWEquals(tsw, fromBinSort);

    long timeSeconds = ts.toEpochSecond();
    if (0 <= timeSeconds && timeSeconds <= Integer.MAX_VALUE) {
      assertEquals(Timestamp.ofEpochSecond(timeSeconds),
        fromIntAndVInts((int) timeSeconds, 0).getTimestamp());

      int nanos = reverseNanos(ts.getNanos());
      assertEquals(ts,
        fromIntAndVInts((int) timeSeconds | (nanos != 0 ? HAS_DECIMAL_MASK : 0),
          nanos).getTimestamp());
    }

    assertEquals(ts.getNanos(), tsw.getNanos());
    assertEquals(getSeconds(ts), tsw.getSeconds());

    // Test various set methods and copy constructors.
    {
      TimestampWritableV2 tsSet1 = new TimestampWritableV2();
      // make the offset non-zero to keep things interesting.
      int offset = Math.abs(ts.hashCode() % 32);
      byte[] shiftedBytes = padBytes(tsBytes, offset);
      tsSet1.set(shiftedBytes, offset);
      assertTSWEquals(tsw, tsSet1);

      TimestampWritableV2 tswShiftedBytes = new TimestampWritableV2(shiftedBytes, offset);
      assertTSWEquals(tsw, tswShiftedBytes);
      assertTSWEquals(tsw, deserializeFromBytes(serializeToBytes(tswShiftedBytes)));
    }

    {
      TimestampWritableV2 tsSet2 = new TimestampWritableV2();
      tsSet2.set(ts);
      assertTSWEquals(tsw, tsSet2);
    }

    {
      TimestampWritableV2 tsSet3 = new TimestampWritableV2();
      tsSet3.set(tsw);
      assertTSWEquals(tsw, tsSet3);
    }

    {
      TimestampWritableV2 tsSet4 = new TimestampWritableV2();
      tsSet4.set(deserTSW);
      assertTSWEquals(tsw, tsSet4);
    }

    double expectedDbl = getSeconds(ts) + 1e-9d * ts.getNanos();
    assertTrue(Math.abs(tsw.getDouble() - expectedDbl) < 1e-10d);

    return deserTSW;
  }

  private static int randomNanos(Random rand, int decimalDigits) {
    // Only keep the most significant decimalDigits digits.
    int nanos = rand.nextInt(BILLION);
    return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits);
  }

  private static int randomNanos(Random rand) {
    return randomNanos(rand, rand.nextInt(10));
  }

  private static void checkTimestampWithAndWithoutNanos(Timestamp ts, int nanos)
      throws IOException {
    serializeDeserializeAndCheckTimestamp(ts);

    ts.setNanos(nanos);
    assertEquals(serializeDeserializeAndCheckTimestamp(ts).getNanos(), nanos);
  }

  private static TimestampWritableV2 fromIntAndVInts(int i, long... vints) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    dos.writeInt(i);
    if ((i & HAS_DECIMAL_MASK) != 0) {
      for (long vi : vints) {
        WritableUtils.writeVLong(dos, vi);
      }
    }
    byte[] bytes = baos.toByteArray();
    TimestampWritableV2 tsw = deserializeFromBytes(bytes);
    assertEquals(toList(bytes), toList(serializeToBytes(tsw)));
    return tsw;
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testReverseNanos() {
    assertEquals(0, reverseNanos(0));
    assertEquals(120000000, reverseNanos(21));
    assertEquals(32100000, reverseNanos(1230));
    assertEquals(5, reverseNanos(500000000));
    assertEquals(987654321, reverseNanos(123456789));
    assertEquals(12345678, reverseNanos(876543210));
  }

  /**
   * Test serializing and deserializing timestamps that can be represented by a number of seconds
   * from 0 to 2147483647 since the UNIX epoch.
   */
  @Test
  @Concurrent(count=4)
  public void testTimestampsWithinPositiveIntRange() throws IOException {
    Random rand = new Random(294722773L);
    for (int i = 0; i < 10000; ++i) {
      long millis = ((long) rand.nextInt(Integer.MAX_VALUE)) * 1000;
      checkTimestampWithAndWithoutNanos(Timestamp.ofEpochMilli(millis), randomNanos(rand));
    }
  }

  private static long randomMillis(long minMillis, long maxMillis, Random rand) {
    return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble());
  }

  /**
   * Test timestamps that don't necessarily fit between 1970 and 2038. This depends on HIVE-4525
   * being fixed.
   */
  @Test
  @Concurrent(count=4)
  public void testTimestampsOutsidePositiveIntRange() throws IOException {
    Random rand = new Random(789149717L);
    for (int i = 0; i < 10000; ++i) {
      long millis = randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand);
      checkTimestampWithAndWithoutNanos(Timestamp.ofEpochMilli(millis), randomNanos(rand));
    }
  }

  @Test
  @Concurrent(count=4)
  public void testTimestampsInFullRange() throws IOException {
    Random rand = new Random(2904974913L);
    for (int i = 0; i < 10000; ++i) {
      checkTimestampWithAndWithoutNanos(Timestamp.ofEpochMilli(rand.nextLong()), randomNanos(rand));
    }
  }

  @Test
  @Concurrent(count=4)
  public void testToFromDouble() {
    Random rand = new Random(294729777L);
    for (int nanosPrecision = 0; nanosPrecision <= 4; ++nanosPrecision) {
      for (int i = 0; i < 10000; ++i) {
        long millis = randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand);
        int nanos = randomNanos(rand, nanosPrecision);
        Timestamp ts = Timestamp.ofEpochMilli(millis, nanos);
        TimestampWritableV2 tsw = new TimestampWritableV2(ts);
        double asDouble = tsw.getDouble();
        int recoveredNanos =
          (int) (Math.round((asDouble - Math.floor(asDouble)) * Math.pow(10, nanosPrecision)) *
            Math.pow(10, 9 - nanosPrecision));
        assertEquals(String.format("Invalid nanosecond part recovered from %f", asDouble),
          nanos, recoveredNanos);
        assertEquals(ts, TimestampUtils.doubleToTimestamp(asDouble));
        // decimalToTimestamp should be consistent with doubleToTimestamp for this level of
        // precision.
        assertEquals(ts, TimestampUtils.decimalToTimestamp(
            HiveDecimal.create(BigDecimal.valueOf(asDouble))));
      }
    }
  }

  private static HiveDecimal timestampToDecimal(Timestamp ts) {
    BigDecimal d = new BigDecimal(getSeconds(ts));
    d = d.add(new BigDecimal(ts.getNanos()).divide(new BigDecimal(BILLION)));
    return HiveDecimal.create(d);
  }

  @Test
  @Concurrent(count=4)
  public void testDecimalToTimestampRandomly() {
    Random rand = new Random(294729777L);
    for (int i = 0; i < 10000; ++i) {
      Timestamp ts = Timestamp.ofEpochMilli(
          randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand));
      ts.setNanos(randomNanos(rand, 9));  // full precision
      assertEquals(ts, TimestampUtils.decimalToTimestamp(timestampToDecimal(ts)));
    }
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testDecimalToTimestampCornerCases() {
    Timestamp ts = Timestamp.ofEpochMilli(parseToMillis("1969-03-04 05:44:33"));
    assertEquals(0, ts.toEpochMilli() % 1000);
    for (int nanos : new int[] { 100000, 900000, 999100000, 999900000 }) {
      ts.setNanos(nanos);
      HiveDecimal d = timestampToDecimal(ts);
      assertEquals(ts, TimestampUtils.decimalToTimestamp(d));
      assertEquals(ts, TimestampUtils.doubleToTimestamp(d.bigDecimalValue().doubleValue()));
    }
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testSerializationFormatDirectly() throws IOException {
    assertEquals("1970-01-01 00:00:00", fromIntAndVInts(0).toString());
    assertEquals("1970-01-01 00:00:01", fromIntAndVInts(1).toString());
    assertEquals("1970-01-01 00:05:00", fromIntAndVInts(300).toString());
    assertEquals("1970-01-01 02:00:00", fromIntAndVInts(7200).toString());
    assertEquals("2000-01-02 03:04:05", fromIntAndVInts(946782245).toString());

    // This won't have a decimal part because the HAS_DECIMAL_MASK bit is not set.
    assertEquals("2000-01-02 03:04:05", fromIntAndVInts(946782245, 3210).toString());

    assertEquals("2000-01-02 03:04:05.0123",
      fromIntAndVInts(946782245 | HAS_DECIMAL_MASK, 3210).toString());

    assertEquals("2038-01-19 03:14:07", fromIntAndVInts(Integer.MAX_VALUE).toString());
    assertEquals("2038-01-19 03:14:07.012345678",
      fromIntAndVInts(Integer.MAX_VALUE | HAS_DECIMAL_MASK,  // this is really just -1
        876543210).toString());

    // Timestamps with a second VInt storing additional bits of the seconds field.
    long seconds = 253392390415L;
    assertEquals("9999-09-08 07:06:55",
      fromIntAndVInts((int) (seconds & 0x7fffffff) | (1 << 31), -1L, seconds >> 31).toString());
    assertEquals("9999-09-08 07:06:55.0123",
      fromIntAndVInts((int) (seconds & 0x7fffffff) | (1 << 31),
                      -3210 - 1, seconds >> 31).toString());
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testMaxSize() {
    // This many bytes are necessary to store the reversed nanoseconds.
    assertEquals(5, WritableUtils.getVIntSize(999999999));
    assertEquals(5, WritableUtils.getVIntSize(-2 - 999999999));

    // Bytes necessary to store extra bits of the second timestamp if storing a timestamp
    // before 1970 or after 2038.
    assertEquals(3, WritableUtils.getVIntSize(Short.MAX_VALUE));
    assertEquals(3, WritableUtils.getVIntSize(Short.MIN_VALUE));

    // Test that MAX_ADDITIONAL_SECONDS_BITS is really the maximum value of the
    // additional bits (beyond 31 bits) of the seconds-since-epoch part of timestamp.
    assertTrue((((long) MAX_ADDITIONAL_SECONDS_BITS) << 31) * 1000 < Long.MAX_VALUE);
    assertTrue((((double) MAX_ADDITIONAL_SECONDS_BITS + 1) * (1L << 31)) * 1000 >
      Long.MAX_VALUE);

    // This is how many bytes we need to store those additonal bits as a VInt.
    assertEquals(4, WritableUtils.getVIntSize(MAX_ADDITIONAL_SECONDS_BITS));

    // Therefore, the maximum total size of a serialized timestamp is 4 + 5 + 4 = 13.
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testMillisToSeconds() {
    assertEquals(0, TimestampUtils.millisToSeconds(0));
    assertEquals(-1, TimestampUtils.millisToSeconds(-1));
    assertEquals(-1, TimestampUtils.millisToSeconds(-999));
    assertEquals(-1, TimestampUtils.millisToSeconds(-1000));
    assertEquals(-2, TimestampUtils.millisToSeconds(-1001));
    assertEquals(-2, TimestampUtils .millisToSeconds(-1999));
    assertEquals(-2, TimestampUtils .millisToSeconds(-2000));
    assertEquals(-3, TimestampUtils .millisToSeconds(-2001));
    assertEquals(-99, TimestampUtils .millisToSeconds(-99000));
    assertEquals(-100, TimestampUtils .millisToSeconds(-99001));
    assertEquals(-100, TimestampUtils .millisToSeconds(-100000));
    assertEquals(1, TimestampUtils .millisToSeconds(1500));
    assertEquals(19, TimestampUtils .millisToSeconds(19999));
    assertEquals(20, TimestampUtils.millisToSeconds(20000));
  }

  private static int compareEqualLengthByteArrays(byte[] a, byte[] b) {
    assertEquals(a.length, b.length);
    for (int i = 0; i < a.length; ++i) {
      if (a[i] != b[i]) {
        return (a[i] & 0xff) - (b[i] & 0xff);
      }
    }
    return 0;
  }

  private static int normalizeComparisonResult(int result) {
    return result < 0 ? -1 : (result > 0 ? 1 : 0);
  }

  @Test
  @Concurrent(count=4)
  @Repeating(repetition=100)
  public void testBinarySortable() {
    Random rand = new Random(5972977L);
    List<TimestampWritableV2> tswList = new ArrayList<TimestampWritableV2>();
    for (int i = 0; i < 50; ++i) {
      Timestamp ts = Timestamp.ofEpochMilli(rand.nextLong(), randomNanos(rand));
      tswList.add(new TimestampWritableV2(ts));
    }
    for (TimestampWritableV2 tsw1 : tswList) {
      byte[] bs1 = tsw1.getBinarySortable();
      for (TimestampWritableV2 tsw2 : tswList) {
        byte[] bs2 = tsw2.getBinarySortable();
        int binaryComparisonResult =
          normalizeComparisonResult(compareEqualLengthByteArrays(bs1, bs2));
        int comparisonResult = normalizeComparisonResult(tsw1.compareTo(tsw2));
        if (binaryComparisonResult != comparisonResult) {
          throw new AssertionError("TimestampWritables " + tsw1 + " and " + tsw2 + " compare as " +
            comparisonResult + " using compareTo but as " + binaryComparisonResult + " using " +
            "getBinarySortable");
        }
      }
    }
  }

  @Test
  public void testSetTimestamp() {
    // one VInt without nanos
    verifySetTimestamp(1000);

    // one VInt with nanos
    verifySetTimestamp(1001);

    // two VInt without nanos
    verifySetTimestamp((long) Integer.MAX_VALUE * 1000 + 1000);

    // two VInt with nanos
    verifySetTimestamp((long) Integer.MAX_VALUE * 1000 + 1234);
  }

  private static void verifySetTimestamp(long time) {
    Timestamp t1 = Timestamp.ofEpochMilli(time);
    TimestampWritableV2 writable = new TimestampWritableV2(t1);
    byte[] bytes = writable.getBytes();
    Timestamp t2 = new Timestamp();
    TimestampWritableV2.setTimestamp(t2, bytes, 0);
    assertEquals(t1, t2);
  }

}
