Package org.apache.flink.table.data.binary

Class BinarySegmentUtils

java.lang.Object
org.apache.flink.table.data.binary.BinarySegmentUtils
@Internal public final class BinarySegmentUtils extends Object
Utilities for binary data segments which heavily uses MemorySegment.

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    static final boolean
    LITTLE_ENDIAN
    Constant that flags the byte order.

  • Method Summary

    Modifier and Type
    Method
    Description
    static byte[]
    allocateReuseBytes(int length)
    Allocate bytes that is only for temporary usage, it should not be stored in somewhere else.
    static char[]
    allocateReuseChars(int length)
     
    static boolean
    bitGet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
    read bit from segments.
    static boolean
    bitGet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
    read bit.
    static void
    bitSet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
    set bit from segments.
    static void
    bitSet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
    set bit.
    static void
    bitUnSet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
    unset bit from segments.
    static void
    bitUnSet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
    unset bit.
    static void
    copyFromBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)
    Copy target segments from source byte[].
    static void
    copyMultiSegmentsToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)
     
    static byte[]
    copyToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)
    Copy segments to target byte[].
    static byte[]
    copyToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
    Copy segments to a new byte[].
    static void
    copyToUnsafe(org.apache.flink.core.memory.MemorySegment[] segments, int offset, Object target, int pointer, int numBytes)
    Copy segments to target unsafe pointer.
    static void
    copyToView(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int sizeInBytes, org.apache.flink.core.memory.DataOutputView target)
    Copy bytes of segments to output view.
    static boolean
    equals(org.apache.flink.core.memory.MemorySegment[] segments1, int offset1, org.apache.flink.core.memory.MemorySegment[] segments2, int offset2, int len)
    Equals two memory segments regions.
    static int
    find(org.apache.flink.core.memory.MemorySegment[] segments1, int offset1, int numBytes1, org.apache.flink.core.memory.MemorySegment[] segments2, int offset2, int numBytes2)
    Find equal segments2 in segments1.
    static boolean
    getBoolean(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get boolean from segments.
    static byte
    getByte(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get byte from segments.
    static byte[]
    getBytes(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int sizeInBytes)
    Maybe not copied, if want copy, please use copyTo.
    static double
    getDouble(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get double from segments.
    static float
    getFloat(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get float from segments.
    static int
    getInt(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get int from segments.
    static long
    getLong(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get long from segments.
    static short
    getShort(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
    get short from segments.
    static int
    hash(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
    hash segments to int.
    static int
    hashByWords(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
    hash segments to int, numBytes must be aligned to 4 bytes.
    static ArrayData
    readArrayData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
    Gets an instance of ArrayData from underlying MemorySegment.
    static byte[]
    readBinary(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int fieldOffset, long variablePartOffsetAndLen)
    Get binary, if len less than 8, will be include in variablePartOffsetAndLen.
    static DecimalData
    readDecimalData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize, int precision, int scale)
    Gets an instance of DecimalData from underlying MemorySegment.
    static MapData
    readMapData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
    Gets an instance of MapData from underlying MemorySegment.
    static <T> RawValueData<T>
    readRawValueData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
    Gets an instance of RawValueData from underlying MemorySegment.
    static RowData
    readRowData(org.apache.flink.core.memory.MemorySegment[] segments, int numFields, int baseOffset, long offsetAndSize)
    Gets an instance of RowData from underlying MemorySegment.
    static StringData
    readStringData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int fieldOffset, long variablePartOffsetAndLen)
    Get binary string, if len less than 8, will be include in variablePartOffsetAndLen.
    static TimestampData
    readTimestampData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndNanos)
    Gets an instance of TimestampData from underlying MemorySegment.
    static void
    setBoolean(org.apache.flink.core.memory.MemorySegment[] segments, int offset, boolean value)
    set boolean from segments.
    static void
    setByte(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte value)
    set byte from segments.
    static void
    setDouble(org.apache.flink.core.memory.MemorySegment[] segments, int offset, double value)
    set double from segments.
    static void
    setFloat(org.apache.flink.core.memory.MemorySegment[] segments, int offset, float value)
    set float from segments.
    static void
    setInt(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int value)
    set int from segments.
    static void
    setLong(org.apache.flink.core.memory.MemorySegment[] segments, int offset, long value)
    set long from segments.
    static void
    setShort(org.apache.flink.core.memory.MemorySegment[] segments, int offset, short value)
    set short from segments.

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

  • Field Details

    • LITTLE_ENDIAN

      public static final boolean LITTLE_ENDIAN
      Constant that flags the byte order.

  • Method Details

    • allocateReuseBytes

      public static byte[] allocateReuseBytes(int length)
      Allocate bytes that is only for temporary usage, it should not be stored in somewhere else. Use a ThreadLocal to reuse bytes to avoid overhead of byte[] new and gc.

      If there are methods that can only accept a byte[], instead of a MemorySegment[] parameter, we can allocate a reuse bytes and copy the MemorySegment data to byte[], then call the method. Such as String deserialization.

    • allocateReuseChars

      public static char[] allocateReuseChars(int length)

    • copyToBytes

      public static byte[] copyToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
      Copy segments to a new byte[].
      Parameters:
      segments - Source segments.
      offset - Source segments offset.
      numBytes - the number bytes to copy.

    • copyToBytes

      public static byte[] copyToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)
      Copy segments to target byte[].
      Parameters:
      segments - Source segments.
      offset - Source segments offset.
      bytes - target byte[].
      bytesOffset - target byte[] offset.
      numBytes - the number bytes to copy.

    • copyMultiSegmentsToBytes

      public static void copyMultiSegmentsToBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)

    • copyToUnsafe

      public static void copyToUnsafe(org.apache.flink.core.memory.MemorySegment[] segments, int offset, Object target, int pointer, int numBytes)
      Copy segments to target unsafe pointer.
      Parameters:
      segments - Source segments.
      offset - The position where the bytes are started to be read from these memory segments.
      target - The unsafe memory to copy the bytes to.
      pointer - The position in the target unsafe memory to copy the chunk to.
      numBytes - the number bytes to copy.

    • copyToView

      public static void copyToView(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int sizeInBytes, org.apache.flink.core.memory.DataOutputView target) throws IOException
      Copy bytes of segments to output view.

      Note: It just copies the data in, not include the length.

      Parameters:
      segments - source segments
      offset - offset for segments
      sizeInBytes - size in bytes
      target - target output view
      Throws:
      IOException

    • copyFromBytes

      public static void copyFromBytes(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte[] bytes, int bytesOffset, int numBytes)
      Copy target segments from source byte[].
      Parameters:
      segments - target segments.
      offset - target segments offset.
      bytes - source byte[].
      bytesOffset - source byte[] offset.
      numBytes - the number bytes to copy.

    • getBytes

      public static byte[] getBytes(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int sizeInBytes)
      Maybe not copied, if want copy, please use copyTo.

    • equals

      public static boolean equals(org.apache.flink.core.memory.MemorySegment[] segments1, int offset1, org.apache.flink.core.memory.MemorySegment[] segments2, int offset2, int len)
      Equals two memory segments regions.
      Parameters:
      segments1 - Segments 1
      offset1 - Offset of segments1 to start equaling
      segments2 - Segments 2
      offset2 - Offset of segments2 to start equaling
      len - Length of the equaled memory region
      Returns:
      true if equal, false otherwise

    • hashByWords

      public static int hashByWords(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
      hash segments to int, numBytes must be aligned to 4 bytes.
      Parameters:
      segments - Source segments.
      offset - Source segments offset.
      numBytes - the number bytes to hash.

    • hash

      public static int hash(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int numBytes)
      hash segments to int.
      Parameters:
      segments - Source segments.
      offset - Source segments offset.
      numBytes - the number bytes to hash.

    • bitUnSet

      public static void bitUnSet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
      unset bit.
      Parameters:
      segment - target segment.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • bitSet

      public static void bitSet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
      set bit.
      Parameters:
      segment - target segment.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • bitGet

      public static boolean bitGet(org.apache.flink.core.memory.MemorySegment segment, int baseOffset, int index)
      read bit.
      Parameters:
      segment - target segment.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • bitUnSet

      public static void bitUnSet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
      unset bit from segments.
      Parameters:
      segments - target segments.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • bitSet

      public static void bitSet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
      set bit from segments.
      Parameters:
      segments - target segments.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • bitGet

      public static boolean bitGet(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int index)
      read bit from segments.
      Parameters:
      segments - target segments.
      baseOffset - bits base offset.
      index - bit index from base offset.

    • getBoolean

      public static boolean getBoolean(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get boolean from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setBoolean

      public static void setBoolean(org.apache.flink.core.memory.MemorySegment[] segments, int offset, boolean value)
      set boolean from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getByte

      public static byte getByte(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get byte from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setByte

      public static void setByte(org.apache.flink.core.memory.MemorySegment[] segments, int offset, byte value)
      set byte from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getInt

      public static int getInt(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get int from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setInt

      public static void setInt(org.apache.flink.core.memory.MemorySegment[] segments, int offset, int value)
      set int from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getLong

      public static long getLong(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get long from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setLong

      public static void setLong(org.apache.flink.core.memory.MemorySegment[] segments, int offset, long value)
      set long from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getShort

      public static short getShort(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get short from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setShort

      public static void setShort(org.apache.flink.core.memory.MemorySegment[] segments, int offset, short value)
      set short from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getFloat

      public static float getFloat(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get float from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setFloat

      public static void setFloat(org.apache.flink.core.memory.MemorySegment[] segments, int offset, float value)
      set float from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • getDouble

      public static double getDouble(org.apache.flink.core.memory.MemorySegment[] segments, int offset)
      get double from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • setDouble

      public static void setDouble(org.apache.flink.core.memory.MemorySegment[] segments, int offset, double value)
      set double from segments.
      Parameters:
      segments - target segments.
      offset - value offset.

    • readDecimalData

      public static DecimalData readDecimalData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize, int precision, int scale)
      Gets an instance of DecimalData from underlying MemorySegment.

    • readTimestampData

      public static TimestampData readTimestampData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndNanos)
      Gets an instance of TimestampData from underlying MemorySegment.
      Parameters:
      segments - the underlying MemorySegments
      baseOffset - the base offset of current instance of TimestampData
      offsetAndNanos - the offset of milli-seconds part and nanoseconds
      Returns:
      an instance of TimestampData

    • readBinary

      public static byte[] readBinary(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int fieldOffset, long variablePartOffsetAndLen)
      Get binary, if len less than 8, will be include in variablePartOffsetAndLen.

      Note: Need to consider the ByteOrder.

      Parameters:
      baseOffset - base offset of composite binary format.
      fieldOffset - absolute start offset of 'variablePartOffsetAndLen'.
      variablePartOffsetAndLen - a long value, real data or offset and len.

    • readStringData

      public static StringData readStringData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, int fieldOffset, long variablePartOffsetAndLen)
      Get binary string, if len less than 8, will be include in variablePartOffsetAndLen.

      Note: Need to consider the ByteOrder.

      Parameters:
      baseOffset - base offset of composite binary format.
      fieldOffset - absolute start offset of 'variablePartOffsetAndLen'.
      variablePartOffsetAndLen - a long value, real data or offset and len.

    • readRawValueData

      public static <T> RawValueData<T> readRawValueData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
      Gets an instance of RawValueData from underlying MemorySegment.

    • readMapData

      public static MapData readMapData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
      Gets an instance of MapData from underlying MemorySegment.

    • readArrayData

      public static ArrayData readArrayData(org.apache.flink.core.memory.MemorySegment[] segments, int baseOffset, long offsetAndSize)
      Gets an instance of ArrayData from underlying MemorySegment.

    • readRowData

      public static RowData readRowData(org.apache.flink.core.memory.MemorySegment[] segments, int numFields, int baseOffset, long offsetAndSize)
      Gets an instance of RowData from underlying MemorySegment.

    • find

      public static int find(org.apache.flink.core.memory.MemorySegment[] segments1, int offset1, int numBytes1, org.apache.flink.core.memory.MemorySegment[] segments2, int offset2, int numBytes2)
      Find equal segments2 in segments1.
      Parameters:
      segments1 - segs to find.
      segments2 - sub segs.
      Returns:
      Return the found offset, return -1 if not find.