The format is laid out as follows:
- *- * 1 11111 1111111111 - * ^ --^-- -----^---- - * sign | |_______ significand - * | - * -- exponent - *- * Half-precision floating points can be useful to save memory and/or - * bandwidth at the expense of range and precision when compared to single-precision - * floating points (float32). - * Ref: https://android.googlesource.com/platform/libcore/+/master/luni/src/main/java/libcore/util/FP16.java - */ -public class Float16 -{ - // Positive 0 of type half-precision float. - static final short POSITIVE_ZERO = (short) 0x0000; - // Negative 0 of type half-precision float. - static final short NEGATIVE_ZERO = (short) 0x8000; - // A Not-a-Number representation of a half-precision float. - static final short NaN = (short) 0x7e00; - // Positive infinity of type half-precision float. - static final short POSITIVE_INFINITY = (short) 0x7c00; - // Negative infinity of type half-precision float. - static final short NEGATIVE_INFINITY = (short) 0xfc00; - // Smallest positive non-zero value a half-precision float may have. - static final short MIN_VALUE = (short) 0x0001; - - // The bitmask to and a number with to obtain the sign bit. - private static final int SIGN_MASK = 0x8000; - // The offset to shift by to obtain the exponent bits. - private static final int EXPONENT_SHIFT = 10; - // The bitmask to and a number shifted by EXPONENT_SHIFT right, to obtain exponent bits. - private static final int SHIFTED_EXPONENT_MASK = 0x1f; - // The bitmask to and a number with to obtain significand bits. - private static final int SIGNIFICAND_MASK = 0x3ff; - // The offset of the exponent from the actual value. - private static final int EXPONENT_BIAS = 15; - // The offset to shift by to obtain the sign bit. - private static final int SIGN_SHIFT = 15; - - private static final int FP32_SIGN_SHIFT = 31; - private static final int FP32_EXPONENT_SHIFT = 23; - private static final int FP32_SHIFTED_EXPONENT_MASK = 0xff; - private static final int FP32_SIGNIFICAND_MASK = 0x7fffff; - private static final int FP32_EXPONENT_BIAS = 127; - private static final int FP32_QNAN_MASK = 0x400000; - private static final int FP32_DENORMAL_MAGIC = 126 << 23; - private static final float FP32_DENORMAL_FLOAT = Float.intBitsToFloat(FP32_DENORMAL_MAGIC); - - /** - * Converts the specified half-precision float value into a - * single-precision float value. The following special cases are handled: - * If the input is NaN, the returned value is Float NaN. - * If the input is POSITIVE_INFINITY or NEGATIVE_INFINITY, the returned value is respectively - * Float POSITIVE_INFINITY or Float NEGATIVE_INFINITY. - * If the input is 0 (positive or negative), the returned value is +/-0.0f. - * Otherwise, the returned value is a normalized single-precision float value. - * - * @param h The half-precision float value to convert to single-precision - * @return A normalized single-precision float value - */ - public static float toFloat(short h) { - int bits = h & 0xffff; - int s = bits & SIGN_MASK; - int e = (bits >>> EXPONENT_SHIFT) & SHIFTED_EXPONENT_MASK; - int m = (bits ) & SIGNIFICAND_MASK; - int outE = 0; - int outM = 0; - if (e == 0) { // Denormal or 0 - if (m != 0) { - // Convert denorm fp16 into normalized fp32 - float o = Float.intBitsToFloat(FP32_DENORMAL_MAGIC + m); - o -= FP32_DENORMAL_FLOAT; - return s == 0 ? o : -o; - } - } else { - outM = m << 13; - if (e == 0x1f) { // Infinite or NaN - outE = 0xff; - if (outM != 0) { // SNaNs are quieted - outM |= FP32_QNAN_MASK; - } - } else { - outE = e - EXPONENT_BIAS + FP32_EXPONENT_BIAS; - } - } - int out = (s << 16) | (outE << FP32_EXPONENT_SHIFT) | outM; - return Float.intBitsToFloat(out); - } - - /** - * Converts the specified single-precision float value into a - * half-precision float value. The following special cases are handled: - * - * If the input is NaN, the returned value is NaN. - * If the input is Float POSITIVE_INFINITY or Float NEGATIVE_INFINITY, - * the returned value is respectively POSITIVE_INFINITY or NEGATIVE_INFINITY. - * If the input is 0 (positive or negative), the returned value is - * POSITIVE_ZERO or NEGATIVE_ZERO. - * If the input is a less than MIN_VALUE, the returned value - * is flushed to POSITIVE_ZERO or NEGATIVE_ZERO. - * If the input is a less than MIN_NORMAL, the returned value - * is a denorm half-precision float. - * Otherwise, the returned value is rounded to the nearest - * representable half-precision float value. - * - * @param f The single-precision float value to convert to half-precision - * @return A half-precision float value - */ - public static short toFloat16(float f) { - int bits = Float.floatToRawIntBits(f); - int s = (bits >>> FP32_SIGN_SHIFT ); - int e = (bits >>> FP32_EXPONENT_SHIFT) & FP32_SHIFTED_EXPONENT_MASK; - int m = (bits ) & FP32_SIGNIFICAND_MASK; - int outE = 0; - int outM = 0; - if (e == 0xff) { // Infinite or NaN - outE = 0x1f; - outM = m != 0 ? 0x200 : 0; - } else { - e = e - FP32_EXPONENT_BIAS + EXPONENT_BIAS; - if (e >= 0x1f) { // Overflow - outE = 0x1f; - } else if (e <= 0) { // Underflow - if (e < -10) { - // The absolute fp32 value is less than MIN_VALUE, flush to +/-0 - } else { - // The fp32 value is a normalized float less than MIN_NORMAL, - // we convert to a denorm fp16 - m = m | 0x800000; - int shift = 14 - e; - outM = m >> shift; - int lowm = m & ((1 << shift) - 1); - int hway = 1 << (shift - 1); - // if above halfway or exactly halfway and outM is odd - if (lowm + (outM & 1) > hway){ - // Round to nearest even - // Can overflow into exponent bit, which surprisingly is OK. - // This increment relies on the +outM in the return statement below - outM++; - } - } - } else { - outE = e; - outM = m >> 13; - // if above halfway or exactly halfway and outM is odd - if ((m & 0x1fff) + (outM & 0x1) > 0x1000) { - // Round to nearest even - // Can overflow into exponent bit, which surprisingly is OK. - // This increment relies on the +outM in the return statement below - outM++; - } - } - } - // The outM is added here as the +1 increments for outM above can - // cause an overflow in the exponent bit which is OK. - return (short) ((s << SIGN_SHIFT) | (outE << EXPONENT_SHIFT) + outM); - } -} diff --git a/parquet-common/src/test/java/org/apache/parquet/util/TestFloat16.java b/parquet-common/src/test/java/org/apache/parquet/util/TestFloat16.java deleted file mode 100644 index d5b61b79a6..0000000000 --- a/parquet-common/src/test/java/org/apache/parquet/util/TestFloat16.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.util; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.apache.parquet.util.Float16.*; - -public class TestFloat16 -{ - @Test - public void testFloat16ToFloat() { - // Zeroes, NaN and infinities - assertEquals(0.0f, toFloat(toFloat16(0.0f)), 0.0f); - assertEquals(-0.0f, toFloat(toFloat16(-0.0f)), 0.0f); - assertEquals(Float.NaN, toFloat(toFloat16(Float.NaN)), 0.0f); - assertEquals(Float.POSITIVE_INFINITY, toFloat(toFloat16(Float.POSITIVE_INFINITY)), 0.0f); - assertEquals(Float.NEGATIVE_INFINITY, toFloat(toFloat16(Float.NEGATIVE_INFINITY)), 0.0f); - // Known values - assertEquals(1.0009765625f, toFloat(toFloat16(1.0009765625f)), 0.0f); - assertEquals(-2.0f, toFloat(toFloat16(-2.0f)), 0.0f); - assertEquals(6.1035156e-5f, toFloat(toFloat16(6.10352e-5f)), 0.0f); // Inexact - assertEquals(65504.0f, toFloat(toFloat16(65504.0f)), 0.0f); - assertEquals(0.33325195f, toFloat(toFloat16(1.0f / 3.0f)), 0.0f); // Inexact - // Denormals (flushed to +/-0) - assertEquals(6.097555e-5f, toFloat(toFloat16(6.09756e-5f)), 0.0f); - assertEquals(5.9604645e-8f, toFloat(toFloat16(5.96046e-8f)), 0.0f); - assertEquals(-6.097555e-5f, toFloat(toFloat16(-6.09756e-5f)), 0.0f); - assertEquals(-5.9604645e-8f, toFloat(toFloat16(-5.96046e-8f)), 0.0f); - } - - @Test - public void testFloatToFloat16() { - // Zeroes, NaN and infinities - assertEquals(POSITIVE_ZERO, toFloat16(0.0f)); - assertEquals(NEGATIVE_ZERO, toFloat16(-0.0f)); - assertEquals(NaN, toFloat16(Float.NaN)); - assertEquals(POSITIVE_INFINITY, toFloat16(Float.POSITIVE_INFINITY)); - assertEquals(NEGATIVE_INFINITY, toFloat16(Float.NEGATIVE_INFINITY)); - // Known values - assertEquals((short) 0x3c01, toFloat16(1.0009765625f)); - assertEquals((short) 0xc000, toFloat16(-2.0f)); - assertEquals((short) 0x0400, toFloat16(6.10352e-5f)); - assertEquals((short) 0x7bff, toFloat16(65504.0f)); - assertEquals((short) 0x3555, toFloat16(1.0f / 3.0f)); - // Subnormals - assertEquals((short) 0x03ff, toFloat16(6.09756e-5f)); - assertEquals(MIN_VALUE, toFloat16(5.96046e-8f)); - assertEquals((short) 0x83ff, toFloat16(-6.09756e-5f)); - assertEquals((short) 0x8001, toFloat16(-5.96046e-8f)); - // Subnormals (flushed to +/-0) - assertEquals(POSITIVE_ZERO, toFloat16(5.96046e-9f)); - assertEquals(NEGATIVE_ZERO, toFloat16(-5.96046e-9f)); - // Test for values that overflow the mantissa bits into exp bits - assertEquals((short) 0x1000, toFloat16(Float.intBitsToFloat(0x39fff000))); - assertEquals((short) 0x0400, toFloat16(Float.intBitsToFloat(0x387fe000))); - // Floats with absolute value above +/-65519 are rounded to +/-inf - // when using round-to-even - assertEquals((short) 0x7bff, toFloat16(65519.0f)); - assertEquals((short) 0x7bff, toFloat16(65519.9f)); - assertEquals(POSITIVE_INFINITY, toFloat16(65520.0f)); - assertEquals(NEGATIVE_INFINITY, toFloat16(-65520.0f)); - // Check if numbers are rounded to nearest even when they - // cannot be accurately represented by Half - assertEquals((short) 0x6800, toFloat16(2049.0f)); - assertEquals((short) 0x6c00, toFloat16(4098.0f)); - assertEquals((short) 0x7000, toFloat16(8196.0f)); - assertEquals((short) 0x7400, toFloat16(16392.0f)); - assertEquals((short) 0x7800, toFloat16(32784.0f)); - } -}