src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java - master - githubmirror/galahad

src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java
388 строк19 KB
Igor Rudenko
8370822: Improve performance for MemorySegment::fill
03 ноя 2025, 17:17
03 ноя 2025, 17:178dbefc5
/*
 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.  Oracle designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Oracle in the LICENSE file that accompanied this code.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

package jdk.internal.foreign;

import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.util.Architecture;
import jdk.internal.util.ArraysSupport;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.annotation.Stable;

import java.lang.foreign.MemorySegment;

/**
 * This class contains optimized bulk operation methods that operate on one or several
 * memory segments.
 * <p>
 * Generally, the methods attempt to work with as-large-as-possible units of memory at
 * a time.
 * <p>
 * It should be noted that when invoking scoped memory access get/set operations, it
 * is imperative from a performance perspective to convey the sharp types from the
 * call site in order for the compiler to pick the correct Unsafe access variant.
 */
public final class SegmentBulkOperations {

    private SegmentBulkOperations() {}

    private static final ScopedMemoryAccess SCOPED_MEMORY_ACCESS = ScopedMemoryAccess.getScopedMemoryAccess();
    private static final long LONG_MASK = ~7L; // The last three bits are zero
    private static final long BYTE_REPLICATOR = 0x0101010101010101L;

    // All the threshold values below MUST be a power of two and should preferably be
    // greater or equal to 2^3.
    private static final int NATIVE_THRESHOLD_FILL = powerOfPropertyOr("fill", 5);
    private static final int NATIVE_THRESHOLD_MISMATCH = powerOfPropertyOr("mismatch", 6);
    private static final int NATIVE_THRESHOLD_COPY = powerOfPropertyOr("copy", 6);

    @ForceInline
    public static MemorySegment fill(AbstractMemorySegmentImpl dst, byte value) {
        dst.checkReadOnly(false);
        if (dst.length == 0) {
            // Implicit state check
            dst.sessionImpl().checkValidState();
        } else if (dst.length < NATIVE_THRESHOLD_FILL) {
            // 0 <= length < FILL_NATIVE_LIMIT : 0...0X...XXXX

            // Handle smaller segments directly without transitioning to native code
            final long u = Byte.toUnsignedLong(value);
            final long longValue = BYTE_REPLICATOR * u;

            int offset = 0;
            // 0...0X...X000
            final int limit = (int) (dst.length & (NATIVE_THRESHOLD_FILL - 8));
            for (; offset < limit; offset += Long.BYTES) {
                SCOPED_MEMORY_ACCESS.putLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, longValue, !Architecture.isLittleEndian());
            }
            int remaining = (int) dst.length - limit;
            // 0...0X00
            if (remaining >= Integer.BYTES) {
                SCOPED_MEMORY_ACCESS.putIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, (int) longValue, !Architecture.isLittleEndian());
                offset += Integer.BYTES;
                remaining -= Integer.BYTES;
            }
            // 0...00X0
            if (remaining >= Short.BYTES) {
                SCOPED_MEMORY_ACCESS.putShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, (short) longValue, !Architecture.isLittleEndian());
                offset += Short.BYTES;
                remaining -= Short.BYTES;
            }
            // 0...000X
            if (remaining == 1) {
                SCOPED_MEMORY_ACCESS.putByte(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, value);
            }
            // We have now fully handled 0...0X...XXXX
        } else {
            // Handle larger segments via native calls
            SCOPED_MEMORY_ACCESS.setMemory(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset(), dst.length, value);
        }
        return dst;
    }

    @ForceInline
    public static void copy(AbstractMemorySegmentImpl src, long srcOffset,
                            AbstractMemorySegmentImpl dst, long dstOffset,
                            long size) {

        Utils.checkNonNegativeIndex(size, "size");
        // Implicit null check for src and dst
        src.checkAccess(srcOffset, size, true);
        dst.checkAccess(dstOffset, size, false);

        if (size <= 0) {
            // Do nothing
        } else if (size < NATIVE_THRESHOLD_COPY && !src.overlaps(dst)) {
            // 0 < size < FILL_NATIVE_LIMIT : 0...0X...XXXX
            //
            // Strictly, we could check for !src.asSlice(srcOffset, size).overlaps(dst.asSlice(dstOffset, size) but
            // this is a bit slower and it likely very unusual there is any difference in the outcome. Also, if there
            // is an overlap, we could tolerate one particular direction of overlap (but not the other).

            // 0...0X...X000
            final int limit = (int) (size & (NATIVE_THRESHOLD_COPY - Long.BYTES));
            int offset = 0;
            for (; offset < limit; offset += Long.BYTES) {
                final long v = SCOPED_MEMORY_ACCESS.getLongUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian());
                SCOPED_MEMORY_ACCESS.putLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian());
            }
            int remaining = (int) size - offset;
            // 0...0X00
            if (remaining >= Integer.BYTES) {
                final int v = SCOPED_MEMORY_ACCESS.getIntUnaligned(src.sessionImpl(), src.unsafeGetBase(),src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian());
                SCOPED_MEMORY_ACCESS.putIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian());
                offset += Integer.BYTES;
                remaining -= Integer.BYTES;
            }
            // 0...00X0
            if (remaining >= Short.BYTES) {
                final short v = SCOPED_MEMORY_ACCESS.getShortUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian());
                SCOPED_MEMORY_ACCESS.putShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian());
                offset += Short.BYTES;
                remaining -= Short.BYTES;
            }
            // 0...000X
            if (remaining == 1) {
                final byte v = SCOPED_MEMORY_ACCESS.getByte(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset + offset);
                SCOPED_MEMORY_ACCESS.putByte(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v);
            }
            // We have now fully handled 0...0X...XXXX
        } else {
            // For larger sizes, the transition to native code pays off
            SCOPED_MEMORY_ACCESS.copyMemory(src.sessionImpl(), dst.sessionImpl(),
                    src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset,
                    dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset, size);
        }
    }

    private static final @Stable int[] POWERS_OF_31 = new int[]{
            0x0000001f, 0x000003c1, 0x0000745f, 0x000e1781,
            0x01b4d89f, 0x34e63b41, 0x67e12cdf, 0x94446f01};

    /**
     * {@return a 32-bit hash value calculated from the content in the provided
     *          {@code segment} between the provided offsets}
     * <p>
     * The method is implemented as a 32-bit polynomial hash function equivalent to:
     * {@snippet lang=java :
     *     final long length = toOffset - fromOffset;
     *     segment.checkBounds(fromOffset, length);
     *     int result = 1;
     *     for (long i = fromOffset; i < toOffset; i++) {
     *         result = 31 * result + segment.get(JAVA_BYTE, i);
     *     }
     *     return result;
     * }
     * but is potentially more performant.
     *
     * @param segment    from which a content hash should be computed
     * @param fromOffset starting offset (inclusive) in the segment
     * @param toOffset   ending offset (non-inclusive) in the segment
     * @throws WrongThreadException if this method is called from a thread {@code T},
     *         such that {@code srcSegment.isAccessibleBy(T) == false}
     * @throws IllegalStateException if the {@linkplain MemorySegment#scope() scope}
     *         associated with {@code segment} is not
     *         {@linkplain MemorySegment.Scope#isAlive() alive}
     * @throws IndexOutOfBoundsException if either {@code fromOffset} or {@code toOffset}
     *                                   are {@code > segment.byteSize}
     * @throws IndexOutOfBoundsException if either {@code fromOffset} or {@code toOffset}
     *                                   are {@code < 0}
     * @throws IndexOutOfBoundsException if {@code toOffset - fromOffset} is {@code < 0}
     */
    @ForceInline
    public static int contentHash(AbstractMemorySegmentImpl segment, long fromOffset, long toOffset) {
        final long length = toOffset - fromOffset;
        segment.checkBounds(fromOffset, length);
        if (length == 0) {
            // The state has to be checked explicitly for zero-length segments
            segment.scope.checkValidState();
            return 1;
        }
        int result = 1;
        final long longBytes = length & LONG_MASK;
        final long limit = fromOffset + longBytes;
        for (; fromOffset < limit; fromOffset += Long.BYTES) {
            long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian());
            result = result * POWERS_OF_31[7]
                    + ((byte) (val >>> 56)) * POWERS_OF_31[6]
                    + ((byte) (val >>> 48)) * POWERS_OF_31[5]
                    + ((byte) (val >>> 40)) * POWERS_OF_31[4]
                    + ((byte) (val >>> 32)) * POWERS_OF_31[3]
                    + ((byte) (val >>> 24)) * POWERS_OF_31[2]
                    + ((byte) (val >>> 16)) * POWERS_OF_31[1]
                    + ((byte) (val >>> 8)) * POWERS_OF_31[0]
                    + ((byte) val);
        }
        int remaining = (int) (length - longBytes);
        // 0...0X00
        if (remaining >= Integer.BYTES) {
            int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian());
            result = result * POWERS_OF_31[3]
                    + ((byte) (val >>> 24)) * POWERS_OF_31[2]
                    + ((byte) (val >>> 16)) * POWERS_OF_31[1]
                    + ((byte) (val >>> 8)) * POWERS_OF_31[0]
                    + ((byte) val);
            fromOffset += Integer.BYTES;
            remaining -= Integer.BYTES;
        }
        // 0...00X0
        if (remaining >= Short.BYTES) {
            short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian());
            result = result * POWERS_OF_31[1]
                    + ((byte) (val >>> 8)) * POWERS_OF_31[0]
                    + ((byte) val);
            fromOffset += Short.BYTES;
            remaining -= Short.BYTES;
        }
        // 0...000X
        if (remaining == 1) {
            byte val = SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset);
            result = result * POWERS_OF_31[0]
                    + val;
        }
        return result;
    }

    @ForceInline
    public static long mismatch(AbstractMemorySegmentImpl src, long srcFromOffset, long srcToOffset,
                                AbstractMemorySegmentImpl dst, long dstFromOffset, long dstToOffset) {
        final long srcBytes = srcToOffset - srcFromOffset;
        final long dstBytes = dstToOffset - dstFromOffset;
        src.checkAccess(srcFromOffset, srcBytes, true);
        dst.checkAccess(dstFromOffset, dstBytes, true);

        final long length = Math.min(srcBytes, dstBytes);
        final boolean srcAndDstBytesDiffer = srcBytes != dstBytes;

        if (length == 0) {
            return srcAndDstBytesDiffer ? 0 : -1;
        } else if (length < NATIVE_THRESHOLD_MISMATCH) {
            return mismatch(src, srcFromOffset, dst, dstFromOffset, 0, (int) length, srcAndDstBytesDiffer);
        } else {
            long i;
            if (SCOPED_MEMORY_ACCESS.getByte(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset) !=
                    SCOPED_MEMORY_ACCESS.getByte(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset)) {
                return 0;
            }
            i = vectorizedMismatchLargeForBytes(src.sessionImpl(), dst.sessionImpl(),
                    src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset,
                    dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset,
                    length);
            if (i >= 0) {
                return i;
            }
            final long remaining = ~i;
            assert remaining < 8 : "remaining greater than 7: " + remaining;
            i = length - remaining;
            return mismatch(src, srcFromOffset + i, dst, dstFromOffset + i, i, (int) remaining, srcAndDstBytesDiffer);
        }
    }

    // Mismatch is handled in chunks of 64 (unroll of eight 8s), 8, 4, 2, and 1 byte(s).
    @ForceInline
    private static long mismatch(AbstractMemorySegmentImpl src, long srcFromOffset,
                                 AbstractMemorySegmentImpl dst, long dstFromOffset,
                                 long start, int length, boolean srcAndDstBytesDiffer) {
        int offset = 0;
        final int limit = length & (NATIVE_THRESHOLD_MISMATCH - 8);
        for (; offset < limit; offset += Long.BYTES) {
            final long s = SCOPED_MEMORY_ACCESS.getLongUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false);
            final long d = SCOPED_MEMORY_ACCESS.getLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false);
            if (s != d) {
                return start + offset + mismatch(s, d);
            }
        }
        int remaining = length - offset;

        // 0...0X00
        if (remaining >= Integer.BYTES) {
            final int s = SCOPED_MEMORY_ACCESS.getIntUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false);
            final int d = SCOPED_MEMORY_ACCESS.getIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false);
            if (s != d) {
                return start + offset + mismatch(s, d);
            }
            offset += Integer.BYTES;
            remaining -= Integer.BYTES;
        }
        // 0...00X0
        if (remaining >= Short.BYTES) {
            final short s = SCOPED_MEMORY_ACCESS.getShortUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false);
            final short d = SCOPED_MEMORY_ACCESS.getShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false);
            if (s != d) {
                return start + offset + mismatch(s, d);
            }
            offset += Short.BYTES;
            remaining -= Short.BYTES;
        }
        // 0...000X
        if (remaining == 1) {
            final byte s = SCOPED_MEMORY_ACCESS.getByte(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset);
            final byte d = SCOPED_MEMORY_ACCESS.getByte(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset);
            if (s != d) {
                return start + offset;
            }
        }
        return srcAndDstBytesDiffer ? (start + length) : -1;
        // We have now fully handled 0...0X...XXXX
    }

    @ForceInline
    private static int mismatch(long first, long second) {
        final long x = first ^ second;
        return Long.numberOfTrailingZeros(x) / 8;
    }

    @ForceInline
    private static int mismatch(int first, int second) {
        final int x = first ^ second;
        return Integer.numberOfTrailingZeros(x) / 8;
    }

    @ForceInline
    private static int mismatch(short first, short second) {
        return ((0xff & first) == (0xff & second)) ? 1 : 0;
    }

    /**
     * Mismatch over long lengths.
     */
    private static long vectorizedMismatchLargeForBytes(MemorySessionImpl aSession, MemorySessionImpl bSession,
                                                        Object a, long aOffset,
                                                        Object b, long bOffset,
                                                        long length) {
        long off = 0;
        long remaining = length;
        int i, size;
        boolean lastSubRange = false;
        while (remaining > 7 && !lastSubRange) {
            if (remaining > Integer.MAX_VALUE) {
                size = Integer.MAX_VALUE;
            } else {
                size = (int) remaining;
                lastSubRange = true;
            }
            i = SCOPED_MEMORY_ACCESS.vectorizedMismatch(aSession, bSession,
                    a, aOffset + off,
                    b, bOffset + off,
                    size, ArraysSupport.LOG2_ARRAY_BYTE_INDEX_SCALE);
            if (i >= 0)
                return off + i;

            i = size - ~i;
            off += i;
            remaining -= i;
        }
        return ~remaining;
    }

    static final String PROPERTY_PATH = "java.lang.foreign.native.threshold.power.";

    // The returned value is in the interval [0, 2^30]
    static int powerOfPropertyOr(String name, int defaultPower) {
        final int power = Integer.getInteger(PROPERTY_PATH + name, defaultPower);
        return 1 << Math.clamp(power, 0, Integer.SIZE - 2);
    }

}