diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/object/ConstantContextSensitiveObject.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/object/ConstantContextSensitiveObject.java index 40a084eb62cf..f6d6924d54c3 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/object/ConstantContextSensitiveObject.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/flow/context/object/ConstantContextSensitiveObject.java @@ -28,6 +28,7 @@ import java.util.List; import com.oracle.graal.pointsto.PointsToAnalysis; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.meta.AnalysisType; import jdk.vm.ci.meta.JavaConstant; @@ -75,6 +76,7 @@ public ConstantContextSensitiveObject(PointsToAnalysis bb, AnalysisType type) { public ConstantContextSensitiveObject(PointsToAnalysis bb, AnalysisType type, JavaConstant constant) { super(bb.getUniverse(), type, AnalysisObjectKind.ConstantContextSensitive); assert bb.trackConcreteAnalysisObjects(type) : type; + assert !(constant instanceof ImageHeapRelocatableConstant) : "relocatable constants have an unknown state and should not be represented by a constant type state: " + constant; this.constant = constant; bb.profileConstantObject(type); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapRelocatableConstant.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapRelocatableConstant.java new file mode 100644 index 000000000000..860cceb951cc --- /dev/null +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapRelocatableConstant.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.pointsto.heap; + +import com.oracle.graal.pointsto.meta.AnalysisType; +import com.oracle.graal.pointsto.util.AnalysisError; + +import jdk.vm.ci.meta.JavaConstant; + +/** + * Used for layered images only. Represents a reference to an object which will be defined in a + * later layer. References to these constants will be patched at execution startup time to point to + * the corresponding object created in a subsequent layer. + */ +public final class ImageHeapRelocatableConstant extends ImageHeapConstant { + + public static final class RelocatableConstantData extends ConstantData { + public final String key; + + RelocatableConstantData(AnalysisType type, String key) { + super(type, null, -1); + this.key = key; + } + } + + private ImageHeapRelocatableConstant(ConstantData constantData, boolean compressed) { + super(constantData, compressed); + } + + @Override + public RelocatableConstantData getConstantData() { + return (RelocatableConstantData) constantData; + } + + public static ImageHeapRelocatableConstant create(AnalysisType type, String key) { + var data = new RelocatableConstantData(type, key); + return new ImageHeapRelocatableConstant(data, false); + } + + @Override + public JavaConstant compress() { + throw AnalysisError.shouldNotReachHere("Unsupported in ImageHeapRelocatableConstant"); + } + + @Override + public JavaConstant uncompress() { + throw AnalysisError.shouldNotReachHere("Unsupported in ImageHeapRelocatableConstant"); + } + + @Override + public ImageHeapConstant forObjectClone() { + throw AnalysisError.shouldNotReachHere("Unsupported in ImageHeapRelocatableConstant"); + } +} diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java index 911b967b0acd..d44633a72b20 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java @@ -159,6 +159,10 @@ private void onInstanceFieldRead(AnalysisField field, AnalysisType type) { for (AnalysisType subtype : type.getSubTypes()) { for (ImageHeapConstant imageHeapConstant : imageHeap.getReachableObjects(subtype)) { FieldScan reason = new FieldScan(field, imageHeapConstant); + if (imageHeapConstant instanceof ImageHeapRelocatableConstant) { + // This constant has no contents to be scanned. + continue; + } ImageHeapInstance imageHeapInstance = (ImageHeapInstance) imageHeapConstant; updateInstanceField(field, imageHeapInstance, reason, null); } @@ -603,6 +607,11 @@ protected void onObjectReachable(ImageHeapConstant imageHeapConstant, ScanReason updateInstanceField(field, imageHeapInstance, new FieldScan(field, imageHeapInstance, reason), onAnalysisModified); } } + } else if (imageHeapConstant instanceof ImageHeapRelocatableConstant) { + /* + * Currently we expect any type registration needed for ImageHeapRelocatableConstants to + * be manually implemented by the user. + */ } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerLoader.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerLoader.java index 068bc04685f7..7e1bf4a9ce2b 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerLoader.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerLoader.java @@ -91,6 +91,7 @@ import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.POSITION_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.PRIMITIVE_ARRAY_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.RETURN_TYPE_TAG; +import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.RELOCATED_CONSTANT_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.SIMULATED_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.SOURCE_FILE_NAME_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.STATIC_OBJECT_FIELDS_TAG; @@ -1098,6 +1099,10 @@ protected ImageHeapConstant getOrCreateConstant(EconomicMap cons Object array = getArray(type.getComponentType().getJavaKind(), primitiveData); addBaseLayerObject(id, objectOffset, () -> new ImageHeapPrimitiveArray(type, null, array, primitiveData.size(), identityHashCode)); } + case RELOCATED_CONSTANT_TAG -> { + String key = get(baseLayerConstant, DATA_TAG); + addBaseLayerObject(id, objectOffset, () -> ImageHeapRelocatableConstant.create(type, key)); + } default -> throw GraalError.shouldNotReachHere("Unknown constant type: " + constantType); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java index 1c01110ff6af..ecbc6471e68a 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java @@ -140,6 +140,7 @@ public class ImageLayerSnapshotUtil { public static final String INSTANCE_TAG = "instance"; public static final String ARRAY_TAG = "array"; public static final String PRIMITIVE_ARRAY_TAG = "primitive array"; + public static final String RELOCATED_CONSTANT_TAG = "relocation constant"; public static final String FIELD_ACCESSED_TAG = "accessed"; public static final String FIELD_READ_TAG = "read"; public static final String FIELD_WRITTEN_TAG = "written"; diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerWriter.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerWriter.java index cef74b53b90e..bc07455be524 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerWriter.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerWriter.java @@ -85,6 +85,7 @@ import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.POSITION_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.PRIMITIVE_ARRAY_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.RETURN_TYPE_TAG; +import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.RELOCATED_CONSTANT_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.SIMULATED_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.SOURCE_FILE_NAME_TAG; import static com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil.STRENGTHENED_GRAPH_TAG; @@ -551,6 +552,10 @@ protected void persistConstant(int parentId, int index, ImageHeapConstant imageH constantMap.put(CONSTANT_TYPE_TAG, PRIMITIVE_ARRAY_TAG); constantMap.put(DATA_TAG, getString(imageHeapPrimitiveArray.getType().getComponentType().getJavaKind(), imageHeapPrimitiveArray.getArray())); } + case ImageHeapRelocatableConstant relocatableConstant -> { + constantMap.put(CONSTANT_TYPE_TAG, RELOCATED_CONSTANT_TAG); + constantMap.put(DATA_TAG, relocatableConstant.getConstantData().key); + } default -> throw AnalysisError.shouldNotReachHere("Unexpected constant type " + imageHeapConstant); } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/ConstantTypeState.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/ConstantTypeState.java index 54ac2cce73eb..1303d7f1b709 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/ConstantTypeState.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/ConstantTypeState.java @@ -31,6 +31,7 @@ import com.oracle.graal.pointsto.BigBang; import com.oracle.graal.pointsto.PointsToAnalysis; import com.oracle.graal.pointsto.flow.context.object.AnalysisObject; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.util.AnalysisError; @@ -49,6 +50,7 @@ public class ConstantTypeState extends SingleTypeState { public ConstantTypeState(PointsToAnalysis bb, AnalysisType type, JavaConstant constant) { super(bb, false, type); assert !bb.analysisPolicy().isContextSensitiveAnalysis() : "The ConstantTypeState is indented to be used with a context insensitive analysis."; + assert !(constant instanceof ImageHeapRelocatableConstant) : "relocatable constants have an unknown state and should not be represented by a constant type state: " + constant; this.constant = constant; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultAnalysisPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultAnalysisPolicy.java index 2a169cc7b5fc..4d8466da972c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultAnalysisPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/typestate/DefaultAnalysisPolicy.java @@ -43,6 +43,7 @@ import com.oracle.graal.pointsto.flow.TypeFlow; import com.oracle.graal.pointsto.flow.context.AnalysisContext; import com.oracle.graal.pointsto.flow.context.object.AnalysisObject; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.AnalysisType; @@ -129,7 +130,14 @@ public AnalysisObject createConstantObject(PointsToAnalysis bb, JavaConstant con } @Override - public ConstantTypeState constantTypeState(PointsToAnalysis bb, JavaConstant constant, AnalysisType exactType) { + public TypeState constantTypeState(PointsToAnalysis bb, JavaConstant constant, AnalysisType exactType) { + if (constant instanceof ImageHeapRelocatableConstant relocatableConstant) { + /* + * ImageHeapRelocatableConstants are placeholder values which will be later replaced + * with an unknown non-null object. + */ + return TypeState.forType(bb, relocatableConstant.getType(), false); + } return new ConstantTypeState(bb, exactType, constant); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java index 93730282fa43..cef3509a1ada 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java @@ -59,8 +59,13 @@ public class ChunkedImageHeapLayouter implements ImageHeapLayouter { * A relocated reference is read-only once relocated, e.g., at runtime. */ private static final int READ_ONLY_RELOCATABLE = READ_ONLY_REGULAR + 1; + /** + * A partition holding objects which must be patched at execution startup by our initialization + * code. This is currently only used within layered images. + */ + private static final int WRITABLE_PATCHED = READ_ONLY_RELOCATABLE + 1; /** A partition holding writable objects. */ - private static final int WRITABLE_REGULAR = READ_ONLY_RELOCATABLE + 1; + private static final int WRITABLE_REGULAR = WRITABLE_PATCHED + 1; /** A partition holding very large writable objects with or without references. */ private static final int WRITABLE_HUGE = WRITABLE_REGULAR + 1; /** @@ -83,6 +88,7 @@ public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset) { this.partitions = new ChunkedImageHeapPartition[PARTITION_COUNT]; this.partitions[READ_ONLY_REGULAR] = new ChunkedImageHeapPartition("readOnly", false, false, alignment, alignment); this.partitions[READ_ONLY_RELOCATABLE] = new ChunkedImageHeapPartition("readOnlyRelocatable", false, false, alignment, alignment); + this.partitions[WRITABLE_PATCHED] = new ChunkedImageHeapPartition("writablePatched", true, false, alignment, alignment); this.partitions[WRITABLE_REGULAR] = new ChunkedImageHeapPartition("writable", true, false, alignment, alignment); this.partitions[WRITABLE_HUGE] = new ChunkedImageHeapPartition("writableHuge", true, true, alignment, alignment); this.partitions[READ_ONLY_HUGE] = new ChunkedImageHeapPartition("readOnlyHuge", false, true, alignment, SubstrateOptions.getPageSize()); @@ -104,14 +110,16 @@ public ChunkedImageHeapPartition[] getPartitions() { } @Override - public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable) { - ChunkedImageHeapPartition partition = choosePartition(info, immutable, relocatable); + public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable, boolean patched) { + ChunkedImageHeapPartition partition = choosePartition(info, immutable, relocatable, patched); info.setHeapPartition(partition); partition.assign(info); } - private ChunkedImageHeapPartition choosePartition(ImageHeapObject info, boolean immutable, boolean hasRelocatables) { - if (immutable) { + private ChunkedImageHeapPartition choosePartition(ImageHeapObject info, boolean immutable, boolean hasRelocatables, boolean patched) { + if (patched) { + return getWritablePatched(); + } else if (immutable) { if (hasRelocatables) { VMError.guarantee(info.getSize() < hugeObjectThreshold, "Objects with relocatable pointers cannot be huge objects"); return getReadOnlyRelocatable(); @@ -177,6 +185,7 @@ private ImageHeapLayoutInfo populateInfoObjects(int dynamicHubCount, int pageSiz } heapInfo.initialize(getReadOnlyRegular().firstObject, getReadOnlyRegular().lastObject, getReadOnlyRelocatable().firstObject, getReadOnlyRelocatable().lastObject, + getWritablePatched().firstObject, getWritablePatched().lastObject, getWritableRegular().firstObject, getWritableRegular().lastObject, getWritableHuge().firstObject, getWritableHuge().lastObject, getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, offsetOfFirstWritableAlignedChunk, offsetOfFirstWritableUnalignedChunk, offsetOfLastWritableUnalignedChunk, dynamicHubCount); @@ -184,7 +193,8 @@ private ImageHeapLayoutInfo populateInfoObjects(int dynamicHubCount, int pageSiz long writableEnd = getWritableHuge().getStartOffset() + getWritableHuge().getSize(); long writableSize = writableEnd - offsetOfFirstWritableAlignedChunk; long imageHeapSize = getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize() - startOffset; - return new ImageHeapLayoutInfo(startOffset, offsetOfFirstWritableAlignedChunk, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), imageHeapSize); + return new ImageHeapLayoutInfo(startOffset, imageHeapSize, offsetOfFirstWritableAlignedChunk, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), + getWritablePatched().getStartOffset(), getWritablePatched().getSize()); } @Override @@ -231,6 +241,10 @@ private ChunkedImageHeapPartition getReadOnlyRelocatable() { return partitions[READ_ONLY_RELOCATABLE]; } + private ChunkedImageHeapPartition getWritablePatched() { + return partitions[WRITABLE_PATCHED]; + } + private ChunkedImageHeapPartition getWritableRegular() { return partitions[WRITABLE_REGULAR]; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 9e753d12c70b..eab6278c9491 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -754,6 +754,9 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc } else if (info.isInReadOnlyRelocatablePartition(ptr)) { log.string("points into the image heap (read-only relocatables)"); return true; + } else if (info.isInWritablePatchedPartition(ptr)) { + log.string("points into the image heap (writable patched)"); + return true; } else if (info.isInWritableRegularPartition(ptr)) { log.string("points into the image heap (writable)"); return true; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index 8aae5b0ad445..fedf181fc44f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -59,6 +59,9 @@ public final class ImageHeapInfo implements MultiLayeredImageSingleton, UnsavedS @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstReadOnlyRelocatableObject; @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastReadOnlyRelocatableObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstWritablePatchedObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastWritablePatchedObject; + @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object firstWritableRegularObject; @UnknownObjectField(availability = AfterHeapLayout.class, canBeNull = true) public Object lastWritableRegularObject; @@ -83,6 +86,7 @@ public ImageHeapInfo() { @SuppressWarnings("hiding") public void initialize(Object firstReadOnlyRegularObject, Object lastReadOnlyRegularObject, Object firstReadOnlyRelocatableObject, Object lastReadOnlyRelocatableObject, + Object firstWritablePatchedObject, Object lastWritablePatchedObject, Object firstWritableRegularObject, Object lastWritableRegularObject, Object firstWritableHugeObject, Object lastWritableHugeObject, Object firstReadOnlyHugeObject, Object lastReadOnlyHugeObject, long offsetOfFirstWritableAlignedChunk, long offsetOfFirstWritableUnalignedChunk, long offsetOfLastWritableUnalignedChunk, int dynamicHubCount) { @@ -93,6 +97,8 @@ public void initialize(Object firstReadOnlyRegularObject, Object lastReadOnlyReg this.lastReadOnlyRegularObject = lastReadOnlyRegularObject; this.firstReadOnlyRelocatableObject = firstReadOnlyRelocatableObject; this.lastReadOnlyRelocatableObject = lastReadOnlyRelocatableObject; + this.firstWritablePatchedObject = firstWritablePatchedObject; + this.lastWritablePatchedObject = lastWritablePatchedObject; this.firstWritableRegularObject = firstWritableRegularObject; this.lastWritableRegularObject = lastWritableRegularObject; this.firstWritableHugeObject = firstWritableHugeObject; @@ -104,15 +110,32 @@ public void initialize(Object firstReadOnlyRegularObject, Object lastReadOnlyReg this.offsetOfLastWritableUnalignedChunk = offsetOfLastWritableUnalignedChunk; this.dynamicHubCount = dynamicHubCount; - // Compute boundaries for checks considering partitions can be empty (first == last == null) - Object firstReadOnlyNonHugeObject = (firstReadOnlyRegularObject != null) ? firstReadOnlyRegularObject : firstReadOnlyRelocatableObject; - Object lastReadOnlyNonHugeObject = (lastReadOnlyRelocatableObject != null) ? lastReadOnlyRelocatableObject : lastReadOnlyRegularObject; - Object firstNonHugeObject = (firstReadOnlyNonHugeObject != null) ? firstReadOnlyNonHugeObject : firstWritableRegularObject; - Object lastNonHugeObject = (lastWritableRegularObject != null) ? lastWritableRegularObject : lastReadOnlyNonHugeObject; - Object firstHugeObject = (firstWritableHugeObject != null) ? firstWritableHugeObject : firstReadOnlyHugeObject; - Object lastHugeObject = (lastReadOnlyHugeObject != null) ? lastReadOnlyHugeObject : lastWritableHugeObject; - this.firstObject = (firstNonHugeObject != null) ? firstNonHugeObject : firstHugeObject; - this.lastObject = (lastHugeObject != null) ? lastHugeObject : lastNonHugeObject; + /* + * Determine first and last objects. Note orderedObject is ordered based on the partition + * layout. Empty partitions will have (first == last == null). + */ + Object[] orderedObjects = { + firstReadOnlyRegularObject, + lastReadOnlyRegularObject, + firstReadOnlyRelocatableObject, + lastReadOnlyRelocatableObject, + firstWritablePatchedObject, + lastWritablePatchedObject, + firstWritableRegularObject, + lastWritableRegularObject, + firstWritableHugeObject, + lastWritableHugeObject, + firstReadOnlyHugeObject, + lastReadOnlyHugeObject + }; + for (Object cur : orderedObjects) { + if (cur != null) { + if (firstObject == null) { + firstObject = cur; + } + lastObject = cur; + } + } } /* @@ -135,6 +158,12 @@ public boolean isInReadOnlyRelocatablePartition(Pointer ptr) { return Word.objectToUntrackedPointer(firstReadOnlyRelocatableObject).belowOrEqual(ptr) && ptr.belowThan(getObjectEnd(lastReadOnlyRelocatableObject)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean isInWritablePatchedPartition(Pointer ptr) { + assert ptr.isNonNull(); + return Word.objectToUntrackedPointer(firstWritablePatchedObject).belowOrEqual(ptr) && ptr.belowThan(getObjectEnd(lastWritablePatchedObject)); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isInWritableRegularPartition(Pointer ptr) { assert ptr.isNonNull(); @@ -199,6 +228,7 @@ private static > T asImageHeapChunk(long offsetInI public void print(Log log) { log.string("ReadOnly: ").zhex(Word.objectToUntrackedPointer(firstReadOnlyRegularObject)).string(" - ").zhex(getObjectEnd(lastReadOnlyRegularObject)).newline(); log.string("ReadOnly Relocatables: ").zhex(Word.objectToUntrackedPointer(firstReadOnlyRelocatableObject)).string(" - ").zhex(getObjectEnd(lastReadOnlyRelocatableObject)).newline(); + log.string("Writeable Patched: ").zhex(Word.objectToUntrackedPointer(firstWritablePatchedObject)).string(" - ").zhex(getObjectEnd(lastWritablePatchedObject)).newline(); log.string("Writable: ").zhex(Word.objectToUntrackedPointer(firstWritableRegularObject)).string(" - ").zhex(getObjectEnd(lastWritableRegularObject)).newline(); log.string("Writable Huge: ").zhex(Word.objectToUntrackedPointer(firstWritableHugeObject)).string(" - ").zhex(getObjectEnd(lastWritableHugeObject)).newline(); log.string("ReadOnly Huge: ").zhex(Word.objectToUntrackedPointer(firstReadOnlyHugeObject)).string(" - ").zhex(getObjectEnd(lastReadOnlyHugeObject)).newline(); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index 5e37e73f6a2d..f5a522796491 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -31,6 +31,8 @@ import static com.oracle.svm.core.Isolates.IMAGE_HEAP_RELOCATABLE_END; import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_BEGIN; import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END; +import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_BEGIN; +import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_END; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_ANY_RELOCATABLE_POINTER; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_BEGIN; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_END; @@ -38,6 +40,8 @@ import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_RELOCATABLE_END; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_BEGIN; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_END; +import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_BEGIN; +import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_END; import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.NEXT_SECTION; import static com.oracle.svm.core.posix.linux.ProcFSSupport.findMapping; import static com.oracle.svm.core.util.PointerUtils.roundDown; @@ -187,12 +191,14 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve Word heapAnyRelocPointer = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_ANY_RELOCATABLE_POINTER)); Word heapWritableBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_WRITEABLE_BEGIN)); Word heapWritableEnd = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_WRITEABLE_END)); + Word heapWritablePatchedBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_WRITEABLE_PATCHED_BEGIN)); + Word heapWritablePatchedEnd = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_WRITEABLE_PATCHED_END)); result = initializeImageHeap(currentHeapStart, remainingSize, curEndPointer, cachedFDPointer, cachedOffsetsPointer, cachedImageHeapRelocationsPtr, MAGIC.get(), heapBegin, heapEnd, heapRelocBegin, heapAnyRelocPointer, heapRelocEnd, - heapWritableBegin, heapWritableEnd); + heapWritablePatchedBegin, heapWritablePatchedEnd, heapWritableBegin, heapWritableEnd); if (result != CEntryPointErrors.NO_ERROR) { freeImageHeap(selfReservedHeapBase); return result; @@ -262,7 +268,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET.get(), CACHED_IMAGE_HEAP_RELOCATIONS.get(), MAGIC.get(), IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(), IMAGE_HEAP_RELOCATABLE_BEGIN.get(), IMAGE_HEAP_A_RELOCATABLE_POINTER.get(), IMAGE_HEAP_RELOCATABLE_END.get(), - IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get()); + IMAGE_HEAP_WRITABLE_PATCHED_BEGIN.get(), IMAGE_HEAP_WRITABLE_PATCHED_END.get(), IMAGE_HEAP_WRITABLE_BEGIN.get(), IMAGE_HEAP_WRITABLE_END.get()); if (result != CEntryPointErrors.NO_ERROR) { freeImageHeap(selfReservedHeapBase); } @@ -275,10 +281,12 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W @Uninterruptible(reason = "Called during isolate initialization.") private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedSize, WordPointer endPointer, WordPointer cachedFd, WordPointer cachedOffsetInFile, WordPointer cachedImageHeapRelocationsPtr, - Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) { + Pointer magicAddress, Word heapBeginSym, Word heapEndSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritablePatchedSym, + Word heapWritablePatchedEndSym, Word heapWritableSym, Word heapWritableEndSym) { assert heapBeginSym.belowOrEqual(heapWritableSym) && heapWritableSym.belowOrEqual(heapWritableEndSym) && heapWritableEndSym.belowOrEqual(heapEndSym); assert heapBeginSym.belowOrEqual(heapRelocsSym) && heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapEndSym); - assert heapAnyRelocPointer.isNull() || (heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym)); + assert heapRelocsSym.belowOrEqual(heapAnyRelocPointer) && heapAnyRelocPointer.belowThan(heapRelocsEndSym); + assert heapRelocsSym.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.belowOrEqual(heapWritablePatchedSym) && heapWritablePatchedSym.belowOrEqual(heapWritableEndSym); SignedWord fd = cachedFd.read(); @@ -313,7 +321,7 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS */ if (fd.equal(CANNOT_OPEN_FD)) { int result = initializeImageHeapWithMremap(imageHeap, imageHeapSize, pageSize, cachedImageHeapRelocationsPtr, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, - heapWritableSym, heapWritableEndSym); + heapWritablePatchedSym, heapWritablePatchedEndSym, heapWritableSym, heapWritableEndSym); if (result == CEntryPointErrors.MREMAP_NOT_SUPPORTED) { /* * MREMAP_DONTUNMAP is not supported, fall back to copying it from memory (the image @@ -331,7 +339,8 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS return CEntryPointErrors.MAP_HEAP_FAILED; } - int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, WordFactory.nullPointer()); + int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, heapWritablePatchedSym, heapWritablePatchedEndSym, + WordFactory.nullPointer()); if (result != CEntryPointErrors.NO_ERROR) { return result; } @@ -341,12 +350,14 @@ private static int initializeImageHeap(Pointer imageHeap, UnsignedWord reservedS @Uninterruptible(reason = "Called during isolate initialization.") private static int initializeImageHeapWithMremap(Pointer imageHeap, UnsignedWord imageHeapSizeInFile, UnsignedWord pageSize, WordPointer cachedImageHeapRelocationsPtr, Word heapBeginSym, - Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritableSym, Word heapWritableEndSym) { + Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, Word heapWritablePatchedSym, Word heapWritablePatchedEndSym, Word heapWritableSym, + Word heapWritableEndSym) { if (!SubstrateOptions.MremapImageHeap.getValue()) { return CEntryPointErrors.MREMAP_NOT_SUPPORTED; } - Pointer cachedImageHeapRelocations = getCachedImageHeapRelocations((Pointer) cachedImageHeapRelocationsPtr, pageSize, heapRelocsSym, heapRelocsEndSym); + Pointer cachedImageHeapRelocations = getCachedImageHeapRelocations((Pointer) cachedImageHeapRelocationsPtr, pageSize, heapRelocsSym, + heapWritablePatchedEndSym); assert cachedImageHeapRelocations.notEqual(0); if (cachedImageHeapRelocations.rawValue() < 0) { return (int) -cachedImageHeapRelocations.rawValue(); // value is a negated error code @@ -359,7 +370,8 @@ private static int initializeImageHeapWithMremap(Pointer imageHeap, UnsignedWord return CEntryPointErrors.MAP_HEAP_FAILED; } - int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, cachedImageHeapRelocations); + int result = copyRelocations(imageHeap, pageSize, heapBeginSym, heapRelocsSym, heapAnyRelocPointer, heapRelocsEndSym, heapWritablePatchedSym, heapWritablePatchedEndSym, + cachedImageHeapRelocations); if (result != CEntryPointErrors.NO_ERROR) { return result; } @@ -374,9 +386,13 @@ private static int initializeImageHeapWithMremap(Pointer imageHeap, UnsignedWord /** * Returns a valid pointer if successful; otherwise, returns a negated * {@linkplain CEntryPointErrors error code}. + * + * It is necessary to cache the image heap relocations because when we remap a file-backed + * memory region, we observe that the "old" virtual address space is reinitialized to its + * original state w/o relocations. */ @Uninterruptible(reason = "Called during isolate initialization.") - private static Pointer getCachedImageHeapRelocations(Pointer cachedImageHeapRelocationsPtr, UnsignedWord pageSize, Word heapRelocsSym, Word heapRelocsEndSym) { + private static Pointer getCachedImageHeapRelocations(Pointer cachedImageHeapRelocationsPtr, UnsignedWord pageSize, Word heapRelocsSym, Word heapWritablePatchedEndSym) { Pointer imageHeapRelocations = cachedImageHeapRelocationsPtr.readWord(0, LocationIdentity.ANY_LOCATION); if (imageHeapRelocations.isNull() || imageHeapRelocations.equal(COPY_RELOCATIONS_IN_PROGRESS)) { if (!cachedImageHeapRelocationsPtr.logicCompareAndSwapWord(0, WordFactory.nullPointer(), COPY_RELOCATIONS_IN_PROGRESS, LocationIdentity.ANY_LOCATION)) { @@ -391,7 +407,7 @@ private static Pointer getCachedImageHeapRelocations(Pointer cachedImageHeapRelo */ Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); - UnsignedWord heapRelocsLength = roundUp(heapRelocsEndSym.subtract(linkedRelocsBoundary), pageSize); + UnsignedWord heapRelocsLength = roundUp(heapWritablePatchedEndSym.subtract(linkedRelocsBoundary), pageSize); int mremapFlags = LinuxLibCHelper.MREMAP_MAYMOVE() | LinuxLibCHelper.MREMAP_DONTUNMAP(); imageHeapRelocations = LinuxLibCHelper.NoTransitions.mremapP(linkedRelocsBoundary, heapRelocsLength, heapRelocsLength, mremapFlags, WordFactory.nullPointer()); @@ -424,33 +440,56 @@ private static Pointer getCachedImageHeapRelocations(Pointer cachedImageHeapRelo @Uninterruptible(reason = "Called during isolate initialization.") private static int copyRelocations(Pointer imageHeap, UnsignedWord pageSize, Word heapBeginSym, Word heapRelocsSym, Pointer heapAnyRelocPointer, Word heapRelocsEndSym, - Pointer cachedRelocsBoundary) { - if (heapAnyRelocPointer.isNonNull()) { - Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); - Pointer sourceRelocsBoundary = cachedRelocsBoundary; - if (sourceRelocsBoundary.isNull()) { - sourceRelocsBoundary = linkedRelocsBoundary; - } - ComparableWord relocatedValue = sourceRelocsBoundary.readWord(heapAnyRelocPointer.subtract(heapRelocsSym)); + Word heapWritablePatchedSym, Word heapWritablePatchedEndSym, Pointer cachedRelocsBoundary) { + Pointer linkedRelocsBoundary = roundDown(heapRelocsSym, pageSize); + Pointer sourceRelocsBoundary = cachedRelocsBoundary.isNonNull() ? cachedRelocsBoundary : linkedRelocsBoundary; + Pointer linkedCopyStart = WordFactory.nullPointer(); + if (heapRelocsEndSym.subtract(heapRelocsSym).isNonNull()) { + /* + * Use a representative pointer to determine whether it is necessary to copy over the + * relocations from the original image, or if it has already been performed during + * build-link time. + */ + ComparableWord relocatedValue = sourceRelocsBoundary.readWord(heapAnyRelocPointer.subtract(linkedRelocsBoundary)); ComparableWord mappedValue = imageHeap.readWord(heapAnyRelocPointer.subtract(heapBeginSym)); if (relocatedValue.notEqual(mappedValue)) { + linkedCopyStart = heapRelocsSym; + } + } + if (linkedCopyStart.isNull() && heapWritablePatchedEndSym.subtract(heapWritablePatchedSym).isNonNull()) { + /* + * When they exist, patched entries always need to be copied over from the original + * image. + */ + linkedCopyStart = heapWritablePatchedSym; + } + if (linkedCopyStart.isNonNull()) { + /* + * A portion of the image heap has relocations either resolved by the dynamic linker or + * (for layered images) manually during our runtime initialization code. To preserve the + * relocations, we must copy this code directly from the original image heap and not + * reload it from disk. + * + * We need to round to page boundaries, so we may copy some extra data which could be + * copy-on-write. Also, we must first remap the pages to avoid loading them from disk + * (only to then overwrite them). + */ + Pointer linkedCopyStartBoundary = roundDown(linkedCopyStart, pageSize); + UnsignedWord copyAlignedSize = roundUp(heapWritablePatchedEndSym.subtract(linkedCopyStartBoundary), pageSize); + Pointer destCopyStartBoundary = imageHeap.add(linkedCopyStartBoundary.subtract(heapBeginSym)); + + Pointer committedCopyBegin = VirtualMemoryProvider.get().commit(destCopyStartBoundary, copyAlignedSize, Access.READ | Access.WRITE); + if (committedCopyBegin.isNull() || committedCopyBegin != destCopyStartBoundary) { + return CEntryPointErrors.PROTECT_HEAP_FAILED; + } + + Pointer sourceCopyStartBoundary = sourceRelocsBoundary.add(linkedCopyStartBoundary.subtract(linkedRelocsBoundary)); + LibC.memcpy(destCopyStartBoundary, sourceCopyStartBoundary, copyAlignedSize); + + // make the relocations read-only again + if (linkedRelocsBoundary.belowOrEqual(heapRelocsEndSym) && heapRelocsEndSym.subtract(heapRelocsSym).isNonNull()) { UnsignedWord relocsAlignedSize = roundUp(heapRelocsEndSym.subtract(linkedRelocsBoundary), pageSize); - Pointer relocsBoundary = imageHeap.add(linkedRelocsBoundary.subtract(heapBeginSym)); - /* - * Addresses were relocated by the dynamic linker, so copy them, but first remap the - * pages to avoid swapping them in from disk. We need to round to page boundaries, - * and so we copy some extra data. - * - * NOTE: while objects with relocations are considered read-only, some of them might - * be part of a chunk with writable objects, in which case the chunk header must - * also be writable, and all the chunk's pages will be unprotected below. - */ - Pointer committedRelocsBegin = VirtualMemoryProvider.get().commit(relocsBoundary, relocsAlignedSize, Access.READ | Access.WRITE); - if (committedRelocsBegin.isNull() || committedRelocsBegin != relocsBoundary) { - return CEntryPointErrors.PROTECT_HEAP_FAILED; - } - LibC.memcpy(relocsBoundary, sourceRelocsBoundary, relocsAlignedSize); - if (VirtualMemoryProvider.get().protect(relocsBoundary, relocsAlignedSize, Access.READ) != 0) { + if (VirtualMemoryProvider.get().protect(destCopyStartBoundary, relocsAlignedSize, Access.READ) != 0) { return CEntryPointErrors.PROTECT_HEAP_FAILED; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java index 5939e00f6c42..290b7e29afb7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java @@ -51,6 +51,8 @@ public class Isolates { public static final String IMAGE_HEAP_A_RELOCATABLE_POINTER_SYMBOL_NAME = "__svm_a_relocatable_pointer"; public static final String IMAGE_HEAP_WRITABLE_BEGIN_SYMBOL_NAME = "__svm_heap_writable_begin"; public static final String IMAGE_HEAP_WRITABLE_END_SYMBOL_NAME = "__svm_heap_writable_end"; + public static final String IMAGE_HEAP_WRITABLE_PATCHED_BEGIN_SYMBOL_NAME = "__svm_heap_writable_patched_begin"; + public static final String IMAGE_HEAP_WRITABLE_PATCHED_END_SYMBOL_NAME = "__svm_heap_writable_patched_end"; public static final CGlobalData IMAGE_HEAP_BEGIN = CGlobalDataFactory.forSymbol(IMAGE_HEAP_BEGIN_SYMBOL_NAME); public static final CGlobalData IMAGE_HEAP_END = CGlobalDataFactory.forSymbol(IMAGE_HEAP_END_SYMBOL_NAME); @@ -59,6 +61,8 @@ public class Isolates { public static final CGlobalData IMAGE_HEAP_A_RELOCATABLE_POINTER = CGlobalDataFactory.forSymbol(IMAGE_HEAP_A_RELOCATABLE_POINTER_SYMBOL_NAME); public static final CGlobalData IMAGE_HEAP_WRITABLE_BEGIN = CGlobalDataFactory.forSymbol(IMAGE_HEAP_WRITABLE_BEGIN_SYMBOL_NAME); public static final CGlobalData IMAGE_HEAP_WRITABLE_END = CGlobalDataFactory.forSymbol(IMAGE_HEAP_WRITABLE_END_SYMBOL_NAME); + public static final CGlobalData IMAGE_HEAP_WRITABLE_PATCHED_BEGIN = CGlobalDataFactory.forSymbol(IMAGE_HEAP_WRITABLE_PATCHED_BEGIN_SYMBOL_NAME); + public static final CGlobalData IMAGE_HEAP_WRITABLE_PATCHED_END = CGlobalDataFactory.forSymbol(IMAGE_HEAP_WRITABLE_PATCHED_END_SYMBOL_NAME); public static final CGlobalData ISOLATE_COUNTER = CGlobalDataFactory.createWord((WordBase) WordFactory.unsigned(1)); /* Only used if SpawnIsolates is disabled. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 1b41bc3d5bb8..555ed246ee69 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode.writeCurrentVMThread; import static com.oracle.svm.core.graal.nodes.WriteHeapBaseNode.writeCurrentVMHeapBase; import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.NO_ALLOCATION; +import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_BEGIN; import static com.oracle.svm.core.util.VMError.shouldNotReachHereUnexpectedInput; import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.HAS_SIDE_EFFECT; @@ -68,6 +69,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.c.function.CEntryPointNativeFunctions; import com.oracle.svm.core.code.CodeInfoTable; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.container.Container; import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode; @@ -78,6 +80,8 @@ import com.oracle.svm.core.heap.ReferenceHandler; import com.oracle.svm.core.heap.ReferenceHandlerThread; import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.imagelayer.ImageLayerSection; import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.log.Log; @@ -107,6 +111,7 @@ import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.graph.Node.ConstantNodeParameter; import jdk.graal.compiler.graph.Node.NodeIntrinsic; +import jdk.graal.compiler.nodes.NamedLocationIdentity; import jdk.graal.compiler.nodes.PauseNode; import jdk.graal.compiler.nodes.extended.ForeignCallNode; import jdk.graal.compiler.nodes.spi.LoweringTool; @@ -116,6 +121,7 @@ import jdk.graal.compiler.replacements.SnippetTemplate.Arguments; import jdk.graal.compiler.replacements.SnippetTemplate.SnippetInfo; import jdk.graal.compiler.replacements.Snippets; +import jdk.graal.compiler.word.Word; import jdk.internal.misc.Unsafe; /** @@ -210,6 +216,57 @@ public static int createIsolateSnippet(CEntryPointCreateIsolateParameters parame return runtimeCallInitializeIsolate(INITIALIZE_ISOLATE, parameters); } + private static final CGlobalData IMAGE_HEAP_PATCHING_STATE = CGlobalDataFactory.createWord(); + + private static final class ImageHeapPatchingState { + static final Word UNINITIALIZED = WordFactory.zero(); + static final Word IN_PROGRESS = WordFactory.unsigned(1); + static final Word SUCCESSFUL = WordFactory.unsigned(2); + } + + @Uninterruptible(reason = "Thread state not yet set up.") + private static void layeredPatchHeapRelativeRelocations() { + Word heapPatchStateAddr = IMAGE_HEAP_PATCHING_STATE.get(); + boolean firstIsolate = heapPatchStateAddr.logicCompareAndSwapWord(0, ImageHeapPatchingState.UNINITIALIZED, ImageHeapPatchingState.IN_PROGRESS, NamedLocationIdentity.OFF_HEAP_LOCATION); + + if (!firstIsolate) { + // spin-wait for first isolate + Word state = heapPatchStateAddr.readWordVolatile(0, LocationIdentity.ANY_LOCATION); + while (state.equal(ImageHeapPatchingState.IN_PROGRESS)) { + PauseNode.pause(); + state = heapPatchStateAddr.readWordVolatile(0, LocationIdentity.ANY_LOCATION); + } + + /* + * Patching has already been successfully completed, nothing needs to be done. + */ + return; + } + + Pointer currentSection = ImageLayerSection.getInitialLayerSection().get(); + Word heapBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN)); + + Word patches = ImageLayerSection.getHeapRelativeRelocationsStart().get(); + int endOffset = Integer.BYTES + (patches.readInt(0) * Integer.BYTES); + + int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); + int offset = Integer.BYTES; + while (offset < endOffset) { + int heapOffset = patches.readInt(offset); + int referenceEncoding = patches.readInt(offset + Integer.BYTES); + + if (referenceSize == 4) { + heapBegin.writeInt(heapOffset, referenceEncoding); + } else { + heapBegin.writeLong(heapOffset, referenceEncoding); + } + + offset += 2 * Integer.BYTES; + } + + heapPatchStateAddr.writeWordVolatile(0, ImageHeapPatchingState.SUCCESSFUL); + } + /** * After parsing the isolate arguments in * {@link IsolateArgumentParser#parse(CEntryPointCreateIsolateParameters, IsolateArguments)} the @@ -230,6 +287,11 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara if (!validPageSize) { return CEntryPointErrors.PAGE_SIZE_CHECK_FAILED; } + + if (ImageLayerBuildingSupport.buildingImageLayer()) { + layeredPatchHeapRelativeRelocations(); + } + CEntryPointCreateIsolateParameters parameters = providedParameters; if (parameters.isNull() || parameters.version() < 1) { parameters = StackValue.get(CEntryPointCreateIsolateParameters.class); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayoutInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayoutInfo.java index 52bf25045ae3..a6ef5657e448 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayoutInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayoutInfo.java @@ -27,6 +27,8 @@ import com.oracle.svm.core.BuildPhaseProvider.AfterHeapLayout; import com.oracle.svm.core.heap.UnknownPrimitiveField; +import jdk.graal.compiler.debug.Assertions; + /** Layout offsets and sizes. All offsets are relative to the heap base. */ public class ImageHeapLayoutInfo { @UnknownPrimitiveField(availability = AfterHeapLayout.class) private final long startOffset; @@ -38,13 +40,24 @@ public class ImageHeapLayoutInfo { @UnknownPrimitiveField(availability = AfterHeapLayout.class) private final long readOnlyRelocatableOffset; @UnknownPrimitiveField(availability = AfterHeapLayout.class) private final long readOnlyRelocatableSize; - public ImageHeapLayoutInfo(long startOffset, long writableOffset, long writableSize, long readOnlyRelocatableOffset, long readOnlyRelocatableSize, long imageHeapSize) { + @UnknownPrimitiveField(availability = AfterHeapLayout.class) private final long writablePatchedOffset; + @UnknownPrimitiveField(availability = AfterHeapLayout.class) private final long writablePatchedSize; + + public ImageHeapLayoutInfo(long startOffset, long imageHeapSize, long writableOffset, long writableSize, long readOnlyRelocatableOffset, long readOnlyRelocatableSize, long writablePatchedOffset, + long writablePatchedSize) { this.startOffset = startOffset; this.imageHeapSize = imageHeapSize; this.writableOffset = writableOffset; this.writableSize = writableSize; this.readOnlyRelocatableOffset = readOnlyRelocatableOffset; this.readOnlyRelocatableSize = readOnlyRelocatableSize; + this.writablePatchedOffset = writablePatchedOffset; + this.writablePatchedSize = writablePatchedSize; + + assert readOnlyRelocatableOffset + readOnlyRelocatableSize <= writablePatchedOffset : Assertions.errorMessage("the writable patched section is placed after the relocations", + readOnlyRelocatableOffset, readOnlyRelocatableSize, writablePatchedOffset); + assert writablePatchedOffset >= writableOffset && ((writablePatchedOffset + writablePatchedSize) <= (writableOffset + writableSize)) : Assertions + .errorMessage("writable patched section must be fully contained in the writable section", writablePatchedOffset, writablePatchedSize, writableOffset, writableSize); } public long getStartOffset() { @@ -74,4 +87,12 @@ public long getReadOnlyRelocatableSize() { public boolean isReadOnlyRelocatable(long offset) { return offset >= readOnlyRelocatableOffset && offset < readOnlyRelocatableOffset + readOnlyRelocatableSize; } + + public long getWritablePatchedOffset() { + return writablePatchedOffset; + } + + public long getWritablePatchedSize() { + return writablePatchedSize; + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayouter.java index b9fd85f31d34..844559e5c7de 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayouter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/ImageHeapLayouter.java @@ -40,7 +40,7 @@ public interface ImageHeapLayouter { /** * Assign an object to the most suitable partition. */ - void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable); + void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable, boolean patched); /** * Places all heap partitions and assigns objects their final offsets. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java index 6866127b1529..cd642b947f26 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java @@ -32,6 +32,7 @@ import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingleton; import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; public abstract class ImageLayerSection implements LayeredImageSingleton { @@ -39,13 +40,16 @@ public abstract class ImageLayerSection implements LayeredImageSingleton { protected final CGlobalData cachedImageFDs; protected final CGlobalData cachedImageHeapOffsets; protected final CGlobalData cachedImageHeapRelocations; + protected final CGlobalData heapRelativeRelocations; protected ImageLayerSection(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets, - CGlobalData cachedImageHeapRelocations) { + CGlobalData cachedImageHeapRelocations, + CGlobalData heapRelativeRelocations) { this.initialSectionStart = initialSectionStart; this.cachedImageFDs = cachedImageFDs; this.cachedImageHeapOffsets = cachedImageHeapOffsets; this.cachedImageHeapRelocations = cachedImageHeapRelocations; + this.heapRelativeRelocations = heapRelativeRelocations; } public enum SectionEntries { @@ -56,6 +60,8 @@ public enum SectionEntries { HEAP_ANY_RELOCATABLE_POINTER, HEAP_WRITEABLE_BEGIN, HEAP_WRITEABLE_END, + HEAP_WRITEABLE_PATCHED_BEGIN, + HEAP_WRITEABLE_PATCHED_END, NEXT_SECTION } @@ -88,6 +94,11 @@ public static CGlobalData getCachedImageHeapRelocations() { return singleton().cachedImageHeapRelocations; } + @Fold + public static CGlobalData getHeapRelativeRelocationsStart() { + return singleton().heapRelativeRelocations; + } + protected abstract int getEntryOffsetInternal(SectionEntries entry); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/PriorLayerMarker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/PriorLayerMarker.java new file mode 100644 index 000000000000..00dc4e911132 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/PriorLayerMarker.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.imagelayer; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +/** + * Marker class which indicates within the shadow heap this object should be replaced with an + * ImageHeapConstant created in a prior layer. + * + * This replacement is performed via an objectToConstant replacer registered by + * {@code CrossLayerConstantRegistryFeature}. + */ +@Platforms(Platform.HOSTED_ONLY.class) +public interface PriorLayerMarker { + + String getKey(); +} diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompiledMethodSupport.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompiledMethodSupport.java index 22c316b99b9d..acb6133742e9 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompiledMethodSupport.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompiledMethodSupport.java @@ -360,7 +360,7 @@ public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receive * We cannot fold simulated values during initial before-analysis graph creation; * however, this runs after analysis has completed. */ - return readValue((AnalysisField) field, receiver, true); + return readValue((AnalysisField) field, receiver, true, false); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderFeature.java index 3cd33eaa6f39..6685997b4886 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderFeature.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.svm.core.BuildPhaseProvider; @@ -34,15 +35,18 @@ import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.imagelayer.CrossLayerConstantRegistry; +import com.oracle.svm.hosted.imagelayer.ObjectToConstantFieldValueTransformer; import com.oracle.svm.hosted.jdk.HostedClassLoaderPackageManagement; import com.oracle.svm.util.ReflectionUtil; import jdk.internal.loader.ClassLoaders; +import jdk.vm.ci.meta.JavaConstant; @AutomaticallyRegisteredFeature public class ClassLoaderFeature implements InternalFeature { private static final String APP_KEY_NAME = "ClassLoader#App"; + private static final String APP_PACKAGE_KEY_NAME = "ClassLoader.Packages#App"; private static final String PLATFORM_KEY_NAME = "ClassLoader#Platform"; private static final String BOOT_KEY_NAME = "ClassLoader#Boot"; @@ -112,10 +116,17 @@ public void duringSetup(DuringSetupAccess access) { packageManager.initialize(nativeImageSystemClassLoader.defaultSystemClassLoader, registry); } + var config = (FeatureImpl.DuringSetupAccessImpl) access; if (ImageLayerBuildingSupport.firstImageBuild()) { access.registerObjectReplacer(this::runtimeClassLoaderObjectReplacer); + if (ImageLayerBuildingSupport.buildingInitialLayer()) { + config.registerObjectReachableCallback(ClassLoader.class, (a1, classLoader, reason) -> { + if (HostedClassLoaderPackageManagement.isGeneratedSerializationClassLoader(classLoader)) { + registry.registerHeapConstant(HostedClassLoaderPackageManagement.getClassLoaderSerializationLookupKey(classLoader), classLoader); + } + }); + } } else { - var config = (FeatureImpl.DuringSetupAccessImpl) access; config.registerObjectToConstantReplacer(obj -> replaceClassLoadersWithLayerConstant(registry, obj)); // relink packages defined in the prior layers config.registerObjectToConstantReplacer(packageManager::replaceWithPriorLayerPackage); @@ -125,32 +136,16 @@ public void duringSetup(DuringSetupAccess access) { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { var packagesField = ReflectionUtil.lookupField(ClassLoader.class, "packages"); - access.registerFieldValueTransformer(packagesField, new FieldValueTransformerWithAvailability() { - - @Override - public boolean isAvailable() { - return BuildPhaseProvider.isHostedUniverseBuilt(); - } - - @Override - public Object transform(Object receiver, Object originalValue) { - assert receiver instanceof ClassLoader : receiver; - assert originalValue instanceof ConcurrentHashMap : "Underlying representation has changed: " + originalValue; - - if (ImageLayerBuildingSupport.buildingInitialLayer()) { - var registry = CrossLayerConstantRegistry.singletonOrNull(); - ClassLoader classLoader = (ClassLoader) receiver; - if (HostedClassLoaderPackageManagement.isGeneratedSerializationClassLoader(classLoader)) { - registry.registerHeapConstant(HostedClassLoaderPackageManagement.getClassLoaderSerializationLookupKey(classLoader), receiver); - } - } - - /* Retrieving initial package state for this class loader. */ - ConcurrentHashMap packages = HostedClassLoaderPackageManagement.singleton().getRegisteredPackages((ClassLoader) receiver); - /* If no package state is available then we must create a clean state. */ - return packages == null ? new ConcurrentHashMap<>() : packages; + var config = (FeatureImpl.BeforeAnalysisAccessImpl) access; + if (!ImageLayerBuildingSupport.buildingImageLayer()) { + access.registerFieldValueTransformer(packagesField, new TraditionalPackageMapTransformer()); + } else { + if (ImageLayerBuildingSupport.buildingInitialLayer()) { + config.registerFieldValueTransformer(packagesField, new InitialLayerPackageMapTransformer()); + } else { + access.registerFieldValueTransformer(packagesField, new ExtensionLayerPackageMapTransformer()); } - }); + } if (ImageLayerBuildingSupport.buildingInitialLayer()) { /* @@ -163,6 +158,81 @@ public Object transform(Object receiver, Object originalValue) { registry.registerHeapConstant(APP_KEY_NAME, nativeImageSystemClassLoader.defaultSystemClassLoader); registry.registerHeapConstant(PLATFORM_KEY_NAME, platformClassLoader); registry.registerHeapConstant(BOOT_KEY_NAME, bootClassLoader); + registry.registerFutureHeapConstant(APP_PACKAGE_KEY_NAME, config.getMetaAccess().lookupJavaType(ConcurrentHashMap.class)); + } + + if (ImageLayerBuildingSupport.buildingApplicationLayer()) { + /* + * We need to scan this because the final package info cannot be installed until after + * analysis has completed. + */ + config.rescanObject(HostedClassLoaderPackageManagement.singleton().getPriorAppClassLoaderPackages()); + } + } + + @Override + public void afterAnalysis(AfterAnalysisAccess access) { + // Register the future constant + if (ImageLayerBuildingSupport.buildingApplicationLayer()) { + var registry = CrossLayerConstantRegistry.singletonOrNull(); + registry.finalizeFutureHeapConstant(APP_PACKAGE_KEY_NAME, HostedClassLoaderPackageManagement.singleton().getAppClassLoaderPackages()); + } + } + + abstract static class PackageMapTransformer implements FieldValueTransformerWithAvailability { + + @Override + public boolean isAvailable() { + return BuildPhaseProvider.isHostedUniverseBuilt(); + } + + Object doTransform(Object receiver, Object originalValue) { + assert receiver instanceof ClassLoader : receiver; + assert originalValue instanceof ConcurrentHashMap : "Underlying representation has changed: " + originalValue; + + /* Retrieving initial package state for this class loader. */ + ConcurrentHashMap packages = HostedClassLoaderPackageManagement.singleton().getRegisteredPackages((ClassLoader) receiver); + /* If no package state is available then we must create a clean state. */ + return packages == null ? new ConcurrentHashMap<>() : packages; + } + } + + static class TraditionalPackageMapTransformer extends PackageMapTransformer { + + @Override + public Object transform(Object receiver, Object originalValue) { + return doTransform(receiver, originalValue); + } + } + + static class InitialLayerPackageMapTransformer extends PackageMapTransformer implements ObjectToConstantFieldValueTransformer { + final CrossLayerConstantRegistry registry = CrossLayerConstantRegistry.singletonOrNull(); + + @Override + public JavaConstant transformToConstant(Object receiver, Object originalValue, Function toConstant) { + if (receiver == nativeImageSystemClassLoader.defaultSystemClassLoader) { + /* + * This map will be assigned within the application layer. Within this layer we + * register a relocatable constant. + */ + return registry.getConstant(APP_PACKAGE_KEY_NAME); + + } + + return toConstant.apply(doTransform(receiver, originalValue)); + } + } + + static class ExtensionLayerPackageMapTransformer implements FieldValueTransformerWithAvailability { + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public Object transform(Object receiver, Object originalValue) { + throw VMError.shouldNotReachHere("No classloaders should be installed in extension layers: %s", receiver); } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java index a04e52a94e40..b7e23ba4be4f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ameta/AnalysisConstantReflectionProvider.java @@ -36,6 +36,7 @@ import com.oracle.graal.pointsto.heap.ImageHeapArray; import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.graal.pointsto.heap.ImageHeapInstance; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.heap.ImageHeapScanner; import com.oracle.graal.pointsto.infrastructure.UniverseMetaAccess; import com.oracle.graal.pointsto.meta.AnalysisField; @@ -204,7 +205,7 @@ private static boolean isExpectedJavaConstant(JavaConstant value) { @Override public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver) { - return readValue((AnalysisField) field, receiver, false); + return readValue((AnalysisField) field, receiver, false, false); } @Override @@ -212,7 +213,7 @@ public JavaConstant boxPrimitive(JavaConstant source) { throw VMError.intentionallyUnimplemented(); } - public JavaConstant readValue(AnalysisField field, JavaConstant receiver, boolean returnSimulatedValues) { + public JavaConstant readValue(AnalysisField field, JavaConstant receiver, boolean returnSimulatedValues, boolean readRelocatableValues) { if (!field.isStatic()) { if (!(receiver instanceof ImageHeapInstance imageHeapInstance) || !field.getDeclaringClass().isAssignableFrom(imageHeapInstance.getType())) { /* @@ -265,6 +266,14 @@ public JavaConstant readValue(AnalysisField field, JavaConstant receiver, boolea HostedValuesProvider hostedValuesProvider = universe.getHostedValuesProvider(); value = heapScanner.createImageHeapConstant(hostedValuesProvider.readFieldValueWithReplacement(field, receiver), ObjectScanner.OtherReason.UNKNOWN); } + + if (!readRelocatableValues && value instanceof ImageHeapRelocatableConstant) { + /* + * During compilation we do not want to fold relocatable constants. However, they must + * be seen during the heap scanning process. + */ + return null; + } return value; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java index 880a86c2cac1..23a57d592d45 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java @@ -104,7 +104,7 @@ public boolean isValueAvailable(AnalysisField field) { @Override public JavaConstant readStaticFieldValue(AnalysisField field) { AnalysisConstantReflectionProvider aConstantReflection = (AnalysisConstantReflectionProvider) this.constantReflection; - return aConstantReflection.readValue(field, null, true); + return aConstantReflection.readValue(field, null, true, true); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageLayerWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageLayerWriter.java index b5da8c77a6d6..68f29cc3371f 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageLayerWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageLayerWriter.java @@ -92,6 +92,7 @@ import com.oracle.svm.hosted.reflect.proxy.ProxySubstitutionType; import com.oracle.svm.util.LogUtils; +import jdk.graal.compiler.debug.Assertions; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; @@ -325,24 +326,24 @@ public void writeInt(String keyName, int value) { @Override public void writeIntList(String keyName, List value) { var previous = keyValueStore.put(keyName, List.of("I(", value)); - assert previous == null : previous; + assert previous == null : Assertions.errorMessage(keyName, previous); } @Override public void writeLong(String keyName, long value) { var previous = keyValueStore.put(keyName, List.of("L", value)); - assert previous == null : previous; + assert previous == null : Assertions.errorMessage(keyName, previous); } @Override public void writeString(String keyName, String value) { var previous = keyValueStore.put(keyName, List.of("S", value)); - assert previous == null : previous; + assert previous == null : Assertions.errorMessage(keyName, previous); } @Override public void writeStringList(String keyName, List value) { var previous = keyValueStore.put(keyName, List.of("S(", value)); - assert previous == null : previous; + assert previous == null : Assertions.errorMessage(keyName, previous); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index 4f3d111db7ae..6f06629b0033 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -499,7 +499,26 @@ public void build(String imageName, DebugContext debug) { objectFile.createDefinedSymbol(heapSection.getName(), heapSection, 0, 0, false, false); long sectionOffsetOfARelocatablePointer = writer.writeHeap(debug, heapSectionBuffer); - assert !SpawnIsolates.getValue() || heapSectionBuffer.getByteBuffer().getLong((int) sectionOffsetOfARelocatablePointer) == 0L; + if (SpawnIsolates.getValue()) { + if (heapLayout.getReadOnlyRelocatableSize() == 0) { + /* + * When there isn't a read only relocation section, the value of the relocatable + * pointer does not matter. We place it at the beginning of the empty + * relocatable section. + */ + assert sectionOffsetOfARelocatablePointer == -1 : sectionOffsetOfARelocatablePointer; + sectionOffsetOfARelocatablePointer = heapLayout.getReadOnlyRelocatableOffset() - heapLayout.getStartOffset(); + + } else { + /* + * During isolate startup this value is read to see if the relocation has + * already been performed; hence, this startup process is contingent on the heap + * value initially being set to 0. + */ + assert heapSectionBuffer.getByteBuffer().getLong((int) sectionOffsetOfARelocatablePointer) == 0L; + } + } + assert ImageLayerBuildingSupport.buildingSharedLayer() || heapLayout.getWritablePatchedSize() == 0 : "The writable patched section is used only in shared layers"; defineDataSymbol(Isolates.IMAGE_HEAP_BEGIN_SYMBOL_NAME, heapSection, 0); defineDataSymbol(Isolates.IMAGE_HEAP_END_SYMBOL_NAME, heapSection, imageHeapSize); @@ -509,6 +528,9 @@ public void build(String imageName, DebugContext debug) { defineDataSymbol(Isolates.IMAGE_HEAP_A_RELOCATABLE_POINTER_SYMBOL_NAME, heapSection, sectionOffsetOfARelocatablePointer); defineDataSymbol(Isolates.IMAGE_HEAP_WRITABLE_BEGIN_SYMBOL_NAME, heapSection, heapLayout.getWritableOffset() - heapLayout.getStartOffset()); defineDataSymbol(Isolates.IMAGE_HEAP_WRITABLE_END_SYMBOL_NAME, heapSection, heapLayout.getWritableOffset() + heapLayout.getWritableSize() - heapLayout.getStartOffset()); + defineDataSymbol(Isolates.IMAGE_HEAP_WRITABLE_PATCHED_BEGIN_SYMBOL_NAME, heapSection, heapLayout.getWritablePatchedOffset() - heapLayout.getStartOffset()); + defineDataSymbol(Isolates.IMAGE_HEAP_WRITABLE_PATCHED_END_SYMBOL_NAME, heapSection, + heapLayout.getWritablePatchedOffset() + heapLayout.getWritablePatchedSize() - heapLayout.getStartOffset()); if (ImageLayerBuildingSupport.buildingExtensionLayer()) { HostedDynamicLayerInfo.singleton().defineSymbolsForPriorLayerMethods(objectFile); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java index 5e0b4651ba11..98adc8289035 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java @@ -50,6 +50,7 @@ import com.oracle.graal.pointsto.heap.HostedValuesProvider; import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.graal.pointsto.heap.ImageHeapInstance; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.heap.ImageHeapScanner; import com.oracle.graal.pointsto.meta.AnalysisUniverse; import com.oracle.graal.pointsto.util.AnalysisError; @@ -114,6 +115,7 @@ public final class NativeImageHeap implements ImageHeap { private final ImageHeapLayouter heapLayouter; private final int minInstanceSize; private final int minArraySize; + private final boolean layeredBuild = ImageLayerBuildingSupport.buildingImageLayer(); /** * A Map from objects at construction-time to native image objects. @@ -285,7 +287,7 @@ Object readInlinedField(HostedField field, JavaConstant receiver) { } private JavaConstant readConstantField(HostedField field, JavaConstant receiver) { - return hConstantReflection.readFieldValue(field, receiver); + return hConstantReflection.readFieldValue(field, receiver, true); } private void addStaticFields() { @@ -303,6 +305,7 @@ private void addStaticFields() { } if (Modifier.isStatic(field.getModifiers()) && field.hasLocation() && field.getType().getStorageKind() == JavaKind.Object && field.isRead()) { assert field.isWritten() || !field.isValueAvailable() || MaterializedConstantFields.singleton().contains(field.wrapped); + /* GR-56699 currently static fields cannot be ImageHeapRelocatableConstants. */ addConstant(readConstantField(field, null), false, field); } } @@ -366,6 +369,7 @@ public void addObject(final Object original, boolean immutableFromParent, final public void addConstant(final JavaConstant constant, boolean immutableFromParent, final Object reason) { assert addObjectsPhase.isAllowed() : "Objects cannot be added at phase: " + addObjectsPhase.toString() + " with reason: " + reason; + VMError.guarantee(!(constant instanceof ImageHeapRelocatableConstant), "ImageHeapRelocatableConstants cannot be added to the image heap: %s", constant); if (constant.getJavaKind().isPrimitive() || constant.isNull() || hMetaAccess.isInstanceOf(constant, WordBase.class)) { return; @@ -477,6 +481,7 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable boolean written = false; boolean references = false; boolean relocatable = false; /* always false when !spawnIsolates() */ + boolean patched = false; /* always false when !layeredBuild */ if (!type.isInstantiated()) { StringBuilder msg = new StringBuilder(); @@ -572,12 +577,20 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable if (field.isRead() && field.isValueAvailable() && !ignoredFields.contains(field)) { if (field.getJavaKind() == JavaKind.Object) { assert field.hasLocation(); - JavaConstant fieldValueConstant = hConstantReflection.readFieldValue(field, constant); + JavaConstant fieldValueConstant = hConstantReflection.readFieldValue(field, constant, true); if (fieldValueConstant.getJavaKind() == JavaKind.Object) { if (spawnIsolates()) { fieldRelocatable = fieldValueConstant instanceof RelocatableConstant; } - recursiveAddConstant(fieldValueConstant, fieldsAreImmutable, info); + if (fieldValueConstant instanceof ImageHeapRelocatableConstant) { + VMError.guarantee(layeredBuild); + /* + * This value will need to be patched during startup. + */ + patched = true; + } else { + recursiveAddConstant(fieldValueConstant, fieldsAreImmutable, info); + } references = true; } } @@ -623,12 +636,9 @@ private void addObjectToImageHeap(final JavaConstant constant, boolean immutable } if (relocatable && !isKnownImmutableConstant(constant)) { - /* The constant comes from the base image and is immutable */ - if (!(constant instanceof ImageHeapConstant imageHeapConstant && imageHeapConstant.isInBaseLayer())) { - VMError.shouldNotReachHere("Object with relocatable pointers must be explicitly immutable: " + hUniverse.getSnippetReflection().asObject(Object.class, constant)); - } + VMError.shouldNotReachHere("Object with relocatable pointers must be explicitly immutable: " + hUniverse.getSnippetReflection().asObject(Object.class, constant)); } - heapLayouter.assignObjectToPartition(info, !written || immutable, references, relocatable); + heapLayouter.assignObjectToPartition(info, !written || immutable, references, relocatable, patched); } /** @@ -823,6 +833,8 @@ public final class ObjectInfo implements ImageHeapObject { private final Object reason; ObjectInfo(ImageHeapConstant constant, long size, HostedClass clazz, int identityHashCode, Object reason) { + assert !(constant instanceof ImageHeapRelocatableConstant) : constant; + this.constant = constant; this.clazz = clazz; this.partition = null; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java index 400fc77c8a91..dde6229f23d3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeapWriter.java @@ -41,6 +41,7 @@ import com.oracle.graal.pointsto.heap.ImageHeapConstant; import com.oracle.graal.pointsto.heap.ImageHeapPrimitiveArray; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.util.AnalysisError; import com.oracle.objectfile.ObjectFile; import com.oracle.svm.core.StaticFieldsSupport; @@ -53,10 +54,12 @@ import com.oracle.svm.core.image.ImageHeapLayoutInfo; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.meta.MethodPointer; +import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.code.CEntryPointLiteralFeature; import com.oracle.svm.hosted.config.DynamicHubLayout; import com.oracle.svm.hosted.config.HybridLayout; import com.oracle.svm.hosted.image.NativeImageHeap.ObjectInfo; +import com.oracle.svm.hosted.imagelayer.CrossLayerConstantRegistryFeature; import com.oracle.svm.hosted.imagelayer.LayeredDispatchTableSupport; import com.oracle.svm.hosted.meta.HostedClass; import com.oracle.svm.hosted.meta.HostedField; @@ -159,18 +162,33 @@ private void writeField(RelocatableBuffer buffer, ObjectInfo fields, HostedField int index = getIndexInBuffer(fields, field.getLocation()); JavaConstant value; try { - value = heap.hConstantReflection.readFieldValue(field, receiver); + value = heap.hConstantReflection.readFieldValue(field, receiver, true); } catch (AnalysisError.TypeNotFoundError ex) { throw NativeImageHeap.reportIllegalType(ex.getType(), info); } - if (value instanceof RelocatableConstant) { + if (value instanceof ImageHeapRelocatableConstant constant) { + int heapOffset = NumUtil.safeToInt(fields.getOffset() + field.getLocation()); + CrossLayerConstantRegistryFeature.singleton().markFutureHeapConstantPatchSite(constant, heapOffset); + fillReferenceWithGarbage(buffer, index); + } else if (value instanceof RelocatableConstant) { addNonDataRelocation(buffer, index, prepareRelocatable(info, value)); } else { write(buffer, index, value, info != null ? info : field); } } + private void fillReferenceWithGarbage(RelocatableBuffer buffer, int index) { + long garbageValue = 0xefefefefefefefefL; + if (referenceSize() == Long.BYTES) { + buffer.getByteBuffer().putLong(index, garbageValue); + } else if (referenceSize() == Integer.BYTES) { + buffer.getByteBuffer().putInt(index, (int) garbageValue); + } else { + throw shouldNotReachHere("Unsupported reference size: " + referenceSize()); + } + } + private void write(RelocatableBuffer buffer, int index, JavaConstant con, Object reason) { if (con.getJavaKind() == JavaKind.Object) { writeReference(buffer, index, con, reason); @@ -338,6 +356,7 @@ private void writeReferenceValue(RelocatableBuffer buffer, int index, long value } private void writeObject(ObjectInfo info, RelocatableBuffer buffer) { + VMError.guarantee(!(info.getConstant() instanceof ImageHeapRelocatableConstant), "ImageHeapRelocationConstants cannot be written to the heap %s", info.getConstant()); /* * Write a reference from the object to its hub. This lives at layout.getHubOffset() from * the object base. diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistry.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistry.java index 0473856f641c..013f8985a3aa 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistry.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistry.java @@ -27,6 +27,7 @@ import org.graalvm.nativeimage.ImageSingletons; import com.oracle.graal.pointsto.heap.ImageHeapConstant; +import com.oracle.graal.pointsto.meta.AnalysisType; /** * Registry to manage cross-layer constant references. @@ -48,7 +49,7 @@ static CrossLayerConstantRegistry singletonOrNull() { ImageHeapConstant getConstant(String keyName); /** - * Checks whether a constant for {@code keyName} was stored in a prior layer. + * Checks whether a constant for {@code keyName} was registered in a prior layer. */ boolean constantExists(String keyName); @@ -64,4 +65,16 @@ static CrossLayerConstantRegistry singletonOrNull() { * {@link #getConstant}. */ void registerHeapConstant(String keyName, Object obj); + + /** + * Registers a key to a constant which will be registered in a later layer via + * {@link #finalizeFutureHeapConstant}. The constant can be retrieved via {@link #getConstant} + * in all layers except the layer which calls {@link #finalizeFutureHeapConstant}. + */ + void registerFutureHeapConstant(String keyName, AnalysisType futureType); + + /** + * Registers a value to associate with a prior {@link #registerFutureHeapConstant} registration. + */ + void finalizeFutureHeapConstant(String keyName, Object obj); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistryFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistryFeature.java index 07f2255f1d94..3b170b85cff3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistryFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/CrossLayerConstantRegistryFeature.java @@ -26,18 +26,23 @@ import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.graalvm.nativeimage.ImageSingletons; import com.oracle.graal.pointsto.heap.ImageHeapConstant; +import com.oracle.graal.pointsto.heap.ImageHeapRelocatableConstant; import com.oracle.graal.pointsto.heap.ImageLayerLoader; +import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.imagelayer.PriorLayerMarker; import com.oracle.svm.core.layeredimagesingleton.FeatureSingleton; import com.oracle.svm.core.layeredimagesingleton.ImageSingletonLoader; import com.oracle.svm.core.layeredimagesingleton.ImageSingletonWriter; @@ -49,15 +54,31 @@ import com.oracle.svm.hosted.image.NativeImageHeap; import com.oracle.svm.hosted.meta.HostedUniverse; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.debug.Assertions; + @AutomaticallyRegisteredFeature public class CrossLayerConstantRegistryFeature implements InternalFeature, FeatureSingleton, CrossLayerConstantRegistry { + private record FutureConstantCandidateInfo(ImageHeapRelocatableConstant constant) { + } + ImageLayerIdTrackingSingleton tracker; - boolean sealed = false; + boolean candidateRegistrySealed = false; + boolean patchingSealed = false; ImageLayerLoader loader; - ConcurrentMap constantCandidates; - ConcurrentMap requiredConstants; + Map constantCandidates; + Map requiredConstants; + Map finalizedFutureConstants; + + private int[] relocationPatches; + int relocationPatchesLength = -1; + + public static CrossLayerConstantRegistryFeature singleton() { + return (CrossLayerConstantRegistryFeature) ImageSingletons.lookup(CrossLayerConstantRegistry.class); + } @Override public boolean isInConfiguration(IsInConfigurationAccess access) { @@ -76,16 +97,36 @@ public void afterRegistration(AfterRegistrationAccess access) { if (ImageLayerBuildingSupport.buildingSharedLayer()) { constantCandidates = new ConcurrentHashMap<>(); requiredConstants = ObservableImageHeapMapProvider.create(); + } else { + constantCandidates = Map.of(); + requiredConstants = Map.of(); + } + if (ImageLayerBuildingSupport.buildingExtensionLayer()) { + finalizedFutureConstants = ObservableImageHeapMapProvider.create(); + } else { + finalizedFutureConstants = Map.of(); } ImageSingletons.add(CrossLayerConstantRegistry.class, this); } @Override public void duringSetup(DuringSetupAccess access) { - loader = ((FeatureImpl.DuringSetupAccessImpl) access).getUniverse().getImageLayerLoader(); - if (ImageLayerBuildingSupport.buildingSharedLayer()) { - LayeredImageHeapObjectAdder.singleton().registerObjectAdder(this::addInitialObjects); + var config = (FeatureImpl.DuringSetupAccessImpl) access; + loader = config.getUniverse().getImageLayerLoader(); + LayeredImageHeapObjectAdder.singleton().registerObjectAdder(this::addInitialObjects); + var registry = CrossLayerConstantRegistry.singletonOrNull(); + config.registerObjectToConstantReplacer(obj -> replacePriorMarkersWithConstant(registry, obj)); + } + + /** + * Replace prior layer markers with {@link ImageHeapConstant}s. + */ + ImageHeapConstant replacePriorMarkersWithConstant(CrossLayerConstantRegistry registry, Object object) { + if (object instanceof PriorLayerMarker priorLayerMarker) { + return registry.getConstant(priorLayerMarker.getKey()); } + + return null; } private void addInitialObjects(NativeImageHeap heap, HostedUniverse hUniverse) { @@ -95,37 +136,123 @@ private void addInitialObjects(NativeImageHeap heap, HostedUniverse hUniverse) { ImageHeapConstant singletonConstant = (ImageHeapConstant) hUniverse.getSnippetReflection().forObject(constant); heap.addConstant(singletonConstant, false, addReason); } - } - @Override - public void beforeImageWrite(BeforeImageWriteAccess access) { - sealed = true; - if (ImageLayerBuildingSupport.buildingSharedLayer()) { + for (Object futureConstant : finalizedFutureConstants.values()) { /* - * Register constant candidates installed in this layer. + * Note we could optimize this to only add the object in the application layer if it was + * used by a prior layer. However, currently it is not worth introducing this + * complication to the code. */ + ImageHeapConstant singletonConstant = (ImageHeapConstant) hUniverse.getSnippetReflection().forObject(futureConstant); + heap.addConstant(singletonConstant, false, addReason); + } + } - var config = (FeatureImpl.BeforeImageWriteAccessImpl) access; - var snippetReflection = config.getHostedUniverse().getSnippetReflection(); - var heap = config.getImage().getHeap(); + @Override + public void afterCompilation(AfterCompilationAccess access) { + candidateRegistrySealed = true; + + /* + * Register all future constants seen in this layer. + */ + constantCandidates.entrySet().stream().filter(e -> e.getValue() instanceof FutureConstantCandidateInfo).forEach(entry -> { + String key = entry.getKey(); + FutureConstantCandidateInfo futureConstant = (FutureConstantCandidateInfo) entry.getValue(); + var constant = futureConstant.constant(); + AnalysisType type = constant.getType(); + tracker.registerFutureTrackingInfo(new FutureTrackingInfo(key, FutureTrackingInfo.State.Type, type.getId(), -1)); + + }); + } - for (var entry : constantCandidates.entrySet()) { + @Override + public void beforeImageWrite(BeforeImageWriteAccess access) { + patchingSealed = true; + var config = (FeatureImpl.BeforeImageWriteAccessImpl) access; + var heap = config.getImage().getHeap(); + var snippetReflection = config.getHostedUniverse().getSnippetReflection(); - var optional = config.getHostedMetaAccess().optionalLookupJavaType(entry.getValue().getClass()); - if (optional.isPresent()) { - var constant = (ImageHeapConstant) snippetReflection.forObject(entry.getValue()); - var objectInfo = heap.getConstantInfo(constant); - if (objectInfo != null && objectInfo.getOffset() >= 0) { - int id = ImageHeapConstant.getConstantID(constant); - tracker.registerKey(entry.getKey(), id); - } + /* + * Register constant candidates installed in this layer. + */ + constantCandidates.entrySet().stream().filter(e -> !(e.getValue() instanceof FutureConstantCandidateInfo)).forEach(entry -> { + Object object = entry.getValue(); + var optional = config.getHostedMetaAccess().optionalLookupJavaType(object.getClass()); + if (optional.isPresent()) { + var constant = (ImageHeapConstant) snippetReflection.forObject(object); + var objectInfo = heap.getConstantInfo(constant); + if (objectInfo != null && objectInfo.getOffset() >= 0) { + int id = ImageHeapConstant.getConstantID(constant); + tracker.registerPriorTrackingInfo(entry.getKey(), id); } } + }); + + assert verifyConstantsInstalled(config); + + /* + * Record the finalized constants in the tracker + */ + for (var entry : finalizedFutureConstants.entrySet()) { + // We know these constants have been installed via addInitialObjects + var futureConstant = (ImageHeapConstant) snippetReflection.forObject(entry.getValue()); + var objectInfo = heap.getConstantInfo(futureConstant); + int id = ImageHeapConstant.getConstantID(futureConstant); + FutureTrackingInfo info = (FutureTrackingInfo) tracker.getTrackingInfo(entry.getKey()); + tracker.updateFutureTrackingInfo(new FutureTrackingInfo(info.key(), FutureTrackingInfo.State.Final, id, NumUtil.safeToInt(objectInfo.getOffset()))); + } - assert verifyConstantsInstalled(config); + if (ImageLayerBuildingSupport.buildingApplicationLayer()) { + generateRelocationPatchArray(); } } + /** + * The relocation patch array contains in integer pairs the (heap offset, reference encoding) + * which need to be patched. The reference encoding uses the appropriate compress encoding + * format. Both the heap offset and reference can be stored in 4-byte integers due to the length + * restrictions of the native-image heap. + * + * has the following format: + * + * 
+     *     ---------------------------------
+     *     | offset | description          |
+     *     | 0      | length               |
+     *     | 4      | heap offset 1        |
+     *     | 8      | reference encoding 1 |
+     *     | 12     | heap offset 2        |
+     *     | 16     | reference encoding 2 |
+     *     |              |
+     *     ---------------------------------
+     *
+     *
+ * + * All patching is performed relative to the initial layer's + * {@link com.oracle.svm.core.Isolates#IMAGE_HEAP_BEGIN}, so we must subtract this offset + * (relative to the image heap start) away from all offsets to patch. + */ + private void generateRelocationPatchArray() { + int shift = ImageSingletons.lookup(CompressEncoding.class).getShift(); + List patchArray = new ArrayList<>(); + int heapBeginOffset = tracker.getImageHeapBeginOffset(); + assert heapBeginOffset >= 0 : "invalid image heap begin offset " + heapBeginOffset; + for (var entry : tracker.futureKeyToPatchingOffsetsMap.entrySet()) { + List offsetsToPatch = entry.getValue(); + FutureTrackingInfo info = (FutureTrackingInfo) tracker.getTrackingInfo(entry.getKey()); + VMError.guarantee(info.state() == FutureTrackingInfo.State.Final, "Invalid future %s", info); + + int referenceEncoding = info.offset() >>> shift; + for (int heapOffset : offsetsToPatch) { + patchArray.add(heapOffset - heapBeginOffset); + patchArray.add(referenceEncoding); + } + } + + relocationPatches = patchArray.stream().mapToInt(Integer::intValue).toArray(); + VMError.guarantee(relocationPatchesLength == relocationPatches.length, "%s %s", relocationPatchesLength, relocationPatches); + } + private boolean verifyConstantsInstalled(FeatureImpl.BeforeImageWriteAccessImpl config) { var snippetReflection = config.getHostedUniverse().getSnippetReflection(); var heap = config.getImage().getHeap(); @@ -139,82 +266,274 @@ private boolean verifyConstantsInstalled(FeatureImpl.BeforeImageWriteAccessImpl return true; } - private void checkSealed() { - VMError.guarantee(!sealed, "Id tracking is sealed"); + @Override + public boolean constantExists(String keyName) { + return tracker.getTrackingInfo(keyName) != null; + } + + @Override + public ImageHeapConstant getConstant(String keyName) { + TrackingInfo idInfo = tracker.getTrackingInfo(keyName); + if (idInfo instanceof PriorTrackingInfo prior) { + return loader.getOrCreateConstant(prior.constantId()); + } + + // Check if a future constant candidate exists + var obj = constantCandidates.get(keyName); + if (obj instanceof FutureConstantCandidateInfo info) { + return info.constant(); + } + + // Retrieve or create constant from FutureTrackingInfo + if (idInfo instanceof FutureTrackingInfo future) { + VMError.guarantee(!finalizedFutureConstants.containsKey(keyName), "Future was finalized in this layer: %s", future); + + if (future.state() != FutureTrackingInfo.State.Type) { + return loader.getOrCreateConstant(future.loaderId()); + } + + // A constant has not been stored in the heap yet. Create and cache a constant candidate + FutureConstantCandidateInfo info = (FutureConstantCandidateInfo) constantCandidates.computeIfAbsent(keyName, (k) -> { + AnalysisType type = loader.getAnalysisType(future.loaderId()); + return new FutureConstantCandidateInfo(ImageHeapRelocatableConstant.create(type, k)); + }); + return info.constant(); + } + + throw VMError.shouldNotReachHere("Missing key: %s", keyName); + } + + private void checkCandidateRegistry() { + VMError.guarantee(!candidateRegistrySealed, "cross layer registry is sealed"); } @Override public void registerConstantCandidate(String keyName, Object obj) { - checkSealed(); - VMError.guarantee(ImageLayerBuildingSupport.buildingSharedLayer(), "This only applies to shared layers"); + VMError.guarantee(keyName != null && obj != null, "CrossLayer constants are expected to be non-null. %s %s", keyName, obj); + checkCandidateRegistry(); var previous = constantCandidates.putIfAbsent(keyName, obj); VMError.guarantee(previous == null && !constantExists(keyName), "This key has been registered before: %s", keyName); } @Override public void registerHeapConstant(String keyName, Object obj) { - VMError.guarantee(obj != null, "CrossLayer constants are expected to be non-null."); registerConstantCandidate(keyName, obj); var previous = requiredConstants.putIfAbsent(keyName, obj); VMError.guarantee(previous == null, "This key has been registered before: %s", keyName); } @Override - public ImageHeapConstant getConstant(String keyName) { - checkSealed(); - Integer id = tracker.getId(keyName); - VMError.guarantee(id != null, "Missing key: %s", keyName); - return loader.getOrCreateConstant(id); + public void registerFutureHeapConstant(String keyName, AnalysisType futureType) { + assert futureType != null; + var futureConstant = new FutureConstantCandidateInfo(ImageHeapRelocatableConstant.create(futureType, keyName)); + registerConstantCandidate(keyName, futureConstant); } @Override - public boolean constantExists(String keyName) { - checkSealed(); - return tracker.getId(keyName) != null; + public void finalizeFutureHeapConstant(String keyName, Object obj) { + checkCandidateRegistry(); + VMError.guarantee(tracker.getTrackingInfo(keyName) instanceof FutureTrackingInfo, "This key was not registered as a future constant %s", keyName); + + var previous = finalizedFutureConstants.putIfAbsent(keyName, obj); + VMError.guarantee(previous == null, "This key has been registered before: %s", keyName); + } + + /** + * Records a spot which at startup time will need to patched to refer to an object defined in a + * subsequent layer. + */ + public void markFutureHeapConstantPatchSite(ImageHeapRelocatableConstant constant, int heapOffset) { + VMError.guarantee(!patchingSealed, "Cross layer patching is sealed"); + var data = constant.getConstantData(); + tracker.registerPatchSite(data.key, heapOffset); + tracker.updateFutureTrackingInfo(new FutureTrackingInfo(data.key, FutureTrackingInfo.State.Relocatable, ImageHeapConstant.getConstantID(constant), -1)); } + + /** + * See {@link #generateRelocationPatchArray} for layout description. + */ + public int computeRelocationPatchesLength() { + patchingSealed = true; + VMError.guarantee(relocationPatchesLength == -1, "called multiple times"); + + int numRelocations = 0; + for (var offsetsToPatch : tracker.futureKeyToPatchingOffsetsMap.values()) { + numRelocations += offsetsToPatch.size() * 2; + } + + relocationPatchesLength = numRelocations; + return relocationPatchesLength; + } + + public int[] getRelocationPatches() { + VMError.guarantee(relocationPatches != null); + return relocationPatches; + } + } class ImageLayerIdTrackingSingleton implements LayeredImageSingleton { - private final Map keyToIdMap = new ConcurrentHashMap<>(); + private static final int UNKNOWN_HEAP_BEGIN_OFFSET = -1; + + private final Map keyToTrackingInfoMap = new HashMap<>(); + final Map> futureKeyToPatchingOffsetsMap = new ConcurrentHashMap<>(); + private final int imageHeapBeginOffset; - Integer getId(String key) { - return keyToIdMap.get(key); + ImageLayerIdTrackingSingleton() { + this(UNKNOWN_HEAP_BEGIN_OFFSET); } - void registerKey(String key, int id) { - var previous = keyToIdMap.putIfAbsent(key, id); + ImageLayerIdTrackingSingleton(int imageHeapBeginOffset) { + this.imageHeapBeginOffset = imageHeapBeginOffset; + } + + TrackingInfo getTrackingInfo(String key) { + return keyToTrackingInfoMap.get(key); + } + + void registerPriorTrackingInfo(String key, int constantId) { + assert key != null && constantId > 0 : Assertions.errorMessage(key, constantId); + var previous = keyToTrackingInfoMap.putIfAbsent(key, new PriorTrackingInfo(constantId)); VMError.guarantee(previous == null, "Two values are registered for this key %s", key); } + public int getImageHeapBeginOffset() { + return imageHeapBeginOffset; + } + + public void registerFutureTrackingInfo(FutureTrackingInfo info) { + updateFutureTrackingInfo0(info, false); + } + + public void updateFutureTrackingInfo(FutureTrackingInfo info) { + updateFutureTrackingInfo0(info, true); + } + + private void updateFutureTrackingInfo0(FutureTrackingInfo info, boolean expectPrevious) { + String key = info.key(); + assert key != null; + var previous = (FutureTrackingInfo) keyToTrackingInfoMap.get(key); + boolean hasPrevious = previous != null; + VMError.guarantee(expectPrevious == hasPrevious, "Mismatch with expectPrevious: %s %s %s", expectPrevious, info, previous); + if (previous != null) { + boolean validState = false; + if (info.state().ordinal() > previous.state().ordinal()) { + validState = previous.key().equals(info.key()); + } else if (info.state() == previous.state()) { + validState = previous.key().equals(info.key()) && info.state() != FutureTrackingInfo.State.Final && previous.loaderId() == info.loaderId(); + } + VMError.guarantee(validState, "Invalid update %s %s", previous, info); + } + + keyToTrackingInfoMap.put(key, info); + } + + void registerPatchSite(String futureKey, int heapIndex) { + List indexes = futureKeyToPatchingOffsetsMap.computeIfAbsent(futureKey, id -> new ArrayList<>()); + indexes.add(heapIndex); + } + @Override public EnumSet getImageBuilderFlags() { return LayeredImageSingletonBuilderFlags.BUILDTIME_ACCESS_ONLY; } + private static String futureKeyPatchKey(String key) { + return String.format("futureOffsetPatches:%s", key); + } + @Override public PersistFlags preparePersist(ImageSingletonWriter writer) { - ArrayList keys = new ArrayList<>(); - ArrayList ids = new ArrayList<>(); - for (var entry : keyToIdMap.entrySet()) { - keys.add(entry.getKey()); - ids.add(entry.getValue()); + ArrayList priorKeys = new ArrayList<>(); + ArrayList priorIds = new ArrayList<>(); + ArrayList futureKeys = new ArrayList<>(); + ArrayList futureStates = new ArrayList<>(); + ArrayList futureLoaderIds = new ArrayList<>(); + ArrayList futureOffsets = new ArrayList<>(); + + for (var entry : keyToTrackingInfoMap.entrySet()) { + String key = entry.getKey(); + TrackingInfo trackingInfo = entry.getValue(); + if (trackingInfo instanceof PriorTrackingInfo prior) { + priorKeys.add(key); + priorIds.add(prior.constantId()); + } else { + FutureTrackingInfo future = (FutureTrackingInfo) trackingInfo; + futureKeys.add(key); + futureStates.add(future.state().ordinal()); + futureLoaderIds.add(future.loaderId()); + if (future.state() == FutureTrackingInfo.State.Final) { + futureOffsets.add(future.offset()); + } + + writer.writeIntList(futureKeyPatchKey(key), futureKeyToPatchingOffsetsMap.getOrDefault(key, List.of())); + } } - writer.writeStringList("keys", keys); - writer.writeIntList("ids", ids); + writer.writeStringList("priorKeys", priorKeys); + writer.writeIntList("priorIds", priorIds); + + writer.writeStringList("futureKeys", futureKeys); + writer.writeIntList("futureStates", futureStates); + writer.writeIntList("futureLoaderIds", futureLoaderIds); + writer.writeIntList("futureOffsets", futureOffsets); + + writer.writeInt("imageHeapBeginOffset", imageHeapBeginOffset == UNKNOWN_HEAP_BEGIN_OFFSET ? Heap.getHeap().getImageHeapOffsetInAddressSpace() : imageHeapBeginOffset); return PersistFlags.CREATE; } @SuppressWarnings("unused") public static Object createFromLoader(ImageSingletonLoader loader) { - var tracker = new ImageLayerIdTrackingSingleton(); + var tracker = new ImageLayerIdTrackingSingleton(loader.readInt("imageHeapBeginOffset")); - Iterator keys = loader.readStringList("keys").iterator(); - Iterator ids = loader.readIntList("ids").iterator(); + Iterator priorKeys = loader.readStringList("priorKeys").iterator(); + Iterator priorIds = loader.readIntList("priorIds").iterator(); - while (keys.hasNext()) { - tracker.registerKey(keys.next(), ids.next()); + while (priorKeys.hasNext()) { + tracker.registerPriorTrackingInfo(priorKeys.next(), priorIds.next()); } + + Iterator futureKeys = loader.readStringList("futureKeys").iterator(); + Iterator futureStates = loader.readIntList("futureStates").iterator(); + Iterator futureLoaderIds = loader.readIntList("futureLoaderIds").iterator(); + Iterator futureOffsets = loader.readIntList("futureOffsets").iterator(); + while (futureKeys.hasNext()) { + String key = futureKeys.next(); + FutureTrackingInfo.State state = FutureTrackingInfo.State.values()[futureStates.next()]; + int loaderId = futureLoaderIds.next(); + int offset = state == FutureTrackingInfo.State.Final ? futureOffsets.next() : -1; + tracker.registerFutureTrackingInfo(new FutureTrackingInfo(key, state, loaderId, offset)); + + List offsetsToPatch = loader.readIntList(futureKeyPatchKey(key)); + offsetsToPatch.forEach(heapOffset -> tracker.registerPatchSite(key, heapOffset)); + } + return tracker; } } + +interface TrackingInfo { +} + +record PriorTrackingInfo(int constantId) implements TrackingInfo { +} + +record FutureTrackingInfo(String key, State state, int loaderId, int offset) implements TrackingInfo { + enum State { + Type, + Relocatable, + Final + } + + public FutureTrackingInfo { + assert key != null && loaderId >= 0 : Assertions.errorMessage(key, loaderId); + switch (state) { + case Type: + case Relocatable: + assert offset == -1 : Assertions.errorMessage(state, offset); + break; + case Final: + assert offset > 0 : Assertions.errorMessage(state, offset); + } + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java index 2d30bf461fce..47610d019641 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ImageLayerSectionFeature.java @@ -60,6 +60,7 @@ import jdk.graal.compiler.core.common.CompressEncoding; import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.word.Word; import jdk.vm.ci.code.Architecture; import jdk.vm.ci.meta.JavaConstant; @@ -69,18 +70,21 @@ * Presently the layout of this section is as follows: * * 
- *  --------------------------------------------------------
- *  |  byte offset | name                                  |
- *  |  0           | heap begin                            |
- *  |  8           | heap end                              |
- *  |  16          | heap relocatable begin                |
- *  |  24          | heap relocatable end                  |
- *  |  32          | heap any relocatable pointer          |
- *  |  40          | heap writable begin                   |
- *  |  48          | heap writable end                     |
- *  |  56          | next layer section (0 if final layer) |
- *  |  64          | cross-layer singleton table start     |
- *  --------------------------------------------------------
+ *  ---------------------------------------------------------
+ *  |  byte offset  | name                                  |
+ *  |  0            | heap begin                            |
+ *  |  8            | heap end                              |
+ *  |  16           | heap relocatable begin                |
+ *  |  24           | heap relocatable end                  |
+ *  |  32           | heap any relocatable pointer          |
+ *  |  40           | heap writable begin                   |
+ *  |  48           | heap writable end                     |
+ *  |  56           | heap writable patched begin           |
+ *  |  64           | heap writable patched end             |
+ *  |  72           | next layer section (0 if final layer) |
+ *  |  80           | cross-layer singleton table start     |
+ *  | (after table) | heap relative patches                 |
+ *  ---------------------------------------------------------
  *
*/ @AutomaticallyRegisteredFeature @@ -95,13 +99,17 @@ public final class ImageLayerSectionFeature implements InternalFeature, FeatureS private static final int HEAP_ANY_RELOCATABLE_POINTER_OFFSET = 32; private static final int HEAP_WRITABLE_BEGIN_OFFSET = 40; private static final int HEAP_WRITABLE_END_OFFSET = 48; - private static final int NEXT_SECTION_OFFSET = 56; - private static final int STATIC_SECTION_SIZE = 64; + private static final int HEAP_WRITABLE_PATCHED_BEGIN_OFFSET = 56; + private static final int HEAP_WRITABLE_PATCHED_END_OFFSET = 64; + private static final int NEXT_SECTION_OFFSET = 72; + private static final int STATIC_SECTION_SIZE = 80; private static final String CACHED_IMAGE_FDS_NAME = "__svm_layer_cached_image_fds"; private static final String CACHED_IMAGE_HEAP_OFFSETS_NAME = "__svm_layer_cached_image_heap_offsets"; private static final String CACHED_IMAGE_HEAP_RELOCATIONS_NAME = "__svm_layer_cached_image_heap_relocations"; + private static final String HEAP_RELATIVE_RELOCATIONS_NAME = "__svm_heap_relative_relocations"; + private static final SignedWord UNASSIGNED_FD = signed(-1); @Override @@ -111,7 +119,7 @@ public boolean isInConfiguration(IsInConfigurationAccess access) { @Override public List> getRequiredFeatures() { - return List.of(HostedDynamicLayerInfoFeature.class, LoadImageSingletonFeature.class); + return List.of(HostedDynamicLayerInfoFeature.class, LoadImageSingletonFeature.class, CrossLayerConstantRegistryFeature.class); } @Override @@ -138,6 +146,7 @@ private static ImageLayerSectionImpl createImageLayerSection() { CGlobalData cachedImageFDs; CGlobalData cachedImageHeapOffsets; CGlobalData cachedImageHeapRelocations; + CGlobalData heapRelativeRelocations = ImageLayerBuildingSupport.buildingInitialLayer() ? CGlobalDataFactory.forSymbol(HEAP_RELATIVE_RELOCATIONS_NAME) : null; if (ImageLayerBuildingSupport.buildingInitialLayer()) { cachedImageFDs = CGlobalDataFactory.forSymbol(CACHED_IMAGE_FDS_NAME); @@ -153,7 +162,7 @@ private static ImageLayerSectionImpl createImageLayerSection() { cachedImageHeapRelocations = null; } - return new ImageLayerSectionImpl(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations); + return new ImageLayerSectionImpl(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations, heapRelativeRelocations); } @Override @@ -178,7 +187,19 @@ public void afterAbstractImageCreation(AfterAbstractImageCreationAccess access) ObjectFile objectFile = ((FeatureImpl.AfterAbstractImageCreationAccessImpl) access).getImage().getObjectFile(); int numSingletonSlots = ImageSingletons.lookup(LoadImageSingletonFeature.class).getConstantToTableSlotMap().size(); - layeredSectionDataByteBuffer = ByteBuffer.wrap(new byte[STATIC_SECTION_SIZE + (ConfigurationValues.getObjectLayout().getReferenceSize() * numSingletonSlots)]).order(ByteOrder.LITTLE_ENDIAN); + int singletonSlotsBufferSize = ConfigurationValues.getObjectLayout().getReferenceSize() * numSingletonSlots; + + int heapPatchInfoStart = STATIC_SECTION_SIZE + singletonSlotsBufferSize; + int heapPatchInfoSize = 0; + if (ImageLayerBuildingSupport.buildingApplicationLayer()) { + /* + * See CrossLayerConstantRegistryFeature#generateRelocationPatchArray for a description + * of the patching table layout. + */ + heapPatchInfoSize = Integer.BYTES + (Integer.BYTES * CrossLayerConstantRegistryFeature.singleton().computeRelocationPatchesLength()); + } + + layeredSectionDataByteBuffer = ByteBuffer.wrap(new byte[heapPatchInfoStart + heapPatchInfoSize]).order(ByteOrder.LITTLE_ENDIAN); layeredSectionData = new BasicProgbitsSectionImpl(layeredSectionDataByteBuffer.array()); // since relocations are present the section it is considered writable @@ -190,12 +211,16 @@ public void afterAbstractImageCreation(AfterAbstractImageCreationAccess access) // this symbol will be defined in the next layer's layer section objectFile.createUndefinedSymbol(nextLayerSymbolName, 0, false); layeredSectionData.markRelocationSite(NEXT_SECTION_OFFSET, ObjectFile.RelocationKind.DIRECT_8, nextLayerSymbolName, 0); + objectFile.createUndefinedSymbol(HEAP_RELATIVE_RELOCATIONS_NAME, 0, false); } else { /* * Note because we provide a byte buffer initialized to zeros nothing needs to be done. * Otherwise, the NEXT_SECTION field entry would need to be cleared within the * application layer. */ + + // Define the heap relative patching start + objectFile.createDefinedSymbol(HEAP_RELATIVE_RELOCATIONS_NAME, layeredImageSection, heapPatchInfoStart, 0, false, true); } // this symbol must be global when it will be read by the prior section @@ -220,6 +245,8 @@ public void beforeImageWrite(BeforeImageWriteAccess access) { layeredSectionData.markRelocationSite(HEAP_ANY_RELOCATABLE_POINTER_OFFSET, ObjectFile.RelocationKind.DIRECT_8, Isolates.IMAGE_HEAP_A_RELOCATABLE_POINTER_SYMBOL_NAME, 0); layeredSectionData.markRelocationSite(HEAP_WRITABLE_BEGIN_OFFSET, ObjectFile.RelocationKind.DIRECT_8, Isolates.IMAGE_HEAP_WRITABLE_BEGIN_SYMBOL_NAME, 0); layeredSectionData.markRelocationSite(HEAP_WRITABLE_END_OFFSET, ObjectFile.RelocationKind.DIRECT_8, Isolates.IMAGE_HEAP_WRITABLE_END_SYMBOL_NAME, 0); + layeredSectionData.markRelocationSite(HEAP_WRITABLE_PATCHED_BEGIN_OFFSET, ObjectFile.RelocationKind.DIRECT_8, Isolates.IMAGE_HEAP_WRITABLE_PATCHED_BEGIN_SYMBOL_NAME, 0); + layeredSectionData.markRelocationSite(HEAP_WRITABLE_PATCHED_END_OFFSET, ObjectFile.RelocationKind.DIRECT_8, Isolates.IMAGE_HEAP_WRITABLE_PATCHED_END_SYMBOL_NAME, 0); var config = (FeatureImpl.BeforeImageWriteAccessImpl) access; @@ -242,13 +269,34 @@ public void beforeImageWrite(BeforeImageWriteAccess access) { } singletonTableOffset += referenceSize; } + + if (ImageLayerBuildingSupport.buildingApplicationLayer()) { + /* + * Currently we place the heap relocation patches exclusively in the application layer. + * + * Note the patch information is always written as an array of ints regardless of the + * reference size. This is allowed because we require the image heap to be less than 2GB + * in size. + * + * See CrossLayerConstantRegistryFeature#generateRelocationPatchArray for a thorough + * description of the patching table layout. + */ + int patchesOffset = singletonTableOffset; + int[] relocationPatches = CrossLayerConstantRegistryFeature.singleton().getRelocationPatches(); + layeredSectionDataByteBuffer.putInt(patchesOffset, relocationPatches.length); + patchesOffset += Integer.BYTES; + for (int value : relocationPatches) { + layeredSectionDataByteBuffer.putInt(patchesOffset, value); + patchesOffset += Integer.BYTES; + } + } } private static class ImageLayerSectionImpl extends ImageLayerSection implements UnsavedSingleton { ImageLayerSectionImpl(CGlobalData initialSectionStart, CGlobalData cachedImageFDs, CGlobalData cachedImageHeapOffsets, - CGlobalData cachedImageHeapRelocations) { - super(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations); + CGlobalData cachedImageHeapRelocations, CGlobalData heapRelativeRelocations) { + super(initialSectionStart, cachedImageFDs, cachedImageHeapOffsets, cachedImageHeapRelocations, heapRelativeRelocations); } @Override @@ -261,6 +309,8 @@ public int getEntryOffsetInternal(SectionEntries entry) { case HEAP_ANY_RELOCATABLE_POINTER -> HEAP_ANY_RELOCATABLE_POINTER_OFFSET; case HEAP_WRITEABLE_BEGIN -> HEAP_WRITABLE_BEGIN_OFFSET; case HEAP_WRITEABLE_END -> HEAP_WRITABLE_END_OFFSET; + case HEAP_WRITEABLE_PATCHED_BEGIN -> HEAP_WRITABLE_PATCHED_BEGIN_OFFSET; + case HEAP_WRITEABLE_PATCHED_END -> HEAP_WRITABLE_PATCHED_END_OFFSET; case NEXT_SECTION -> NEXT_SECTION_OFFSET; }; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ObjectToConstantFieldValueTransformer.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ObjectToConstantFieldValueTransformer.java new file mode 100644 index 000000000000..0a6b61976224 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/ObjectToConstantFieldValueTransformer.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.imagelayer; + +import java.util.function.Function; + +import com.oracle.svm.core.fieldvaluetransformer.FieldValueTransformerWithAvailability; +import com.oracle.svm.core.util.VMError; + +import jdk.vm.ci.meta.JavaConstant; + +/** + * Special field value transformer which, instead of returning an object, returns an + * {@link JavaConstant} of the transformed value. + */ +public interface ObjectToConstantFieldValueTransformer extends FieldValueTransformerWithAvailability { + + JavaConstant transformToConstant(Object receiver, Object originalValue, Function toConstant); + + @Override + default Object transform(Object receiver, Object originalValue) { + throw VMError.intentionallyUnimplemented(); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/HostedClassLoaderPackageManagement.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/HostedClassLoaderPackageManagement.java index f7e30a012d43..3035780d678a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/HostedClassLoaderPackageManagement.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/HostedClassLoaderPackageManagement.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.imagelayer.PriorLayerMarker; import com.oracle.svm.core.jdk.Target_java_lang_ClassLoader; import com.oracle.svm.core.layeredimagesingleton.ImageSingletonLoader; import com.oracle.svm.core.layeredimagesingleton.ImageSingletonWriter; @@ -218,6 +219,8 @@ public void registerPackage(ClassLoader runtimeClassLoader, String packageName, * We must register this package so that it can be relinked in subsequent layers. */ registry.registerHeapConstant(generateKeyName(packageValue.getName()), packageValue); + + VMError.guarantee(packageValue.getName().equals(packageName), "Package name is different from package value's name: %s %s", packageName, packageValue); } } } @@ -251,6 +254,49 @@ public ConcurrentHashMap getRegisteredPackages(ClassLoader clas return registeredPackages.get(classLoader); } + /** + * Returns a map containing all packages installed by prior layers, as well as the entries + * installed in this layer. + */ + public ConcurrentHashMap getAppClassLoaderPackages() { + VMError.guarantee(BuildPhaseProvider.isAnalysisFinished(), "Packages are stable only after analysis."); + VMError.guarantee(ImageLayerBuildingSupport.lastImageBuild(), "AppClassLoader's Packages are only fully available in the application layer"); + + var finalAppMap = getPriorAppClassLoaderPackages(); + var currentPackages = registeredPackages.get(appClassLoader); + if (currentPackages != null) { + for (var entry : currentPackages.entrySet()) { + var previous = finalAppMap.put(entry.getKey(), entry.getValue()); + assert previous == null : Assertions.errorMessage(entry.getKey(), previous); + } + } + + return finalAppMap; + } + + /** + * Returns a concurrent hashmap containing the package map produced via the prior layer. Note at + * runtime this map's keys will all be of type {@link Package}. We use a + * {@link PriorLayerMarker} to perform this transformation. + */ + public ConcurrentHashMap getPriorAppClassLoaderPackages() { + ConcurrentHashMap finalAppMap = new ConcurrentHashMap<>(); + for (String name : priorAppPackageNames) { + var previous = finalAppMap.put(name, new PriorLayerPackageReference(generateKeyName(name))); + assert previous == null : Assertions.errorMessage(name, previous); + } + return finalAppMap; + } + + private record PriorLayerPackageReference(String key) implements PriorLayerMarker { + + @Override + public String getKey() { + return key; + } + + } + public void initialize(ClassLoader newAppClassLoader, CrossLayerConstantRegistry newRegistry) { assert appClassLoader == null && newAppClassLoader != null : Assertions.errorMessage(appClassLoader, newAppClassLoader); assert registry == null && newRegistry != null : Assertions.errorMessage(registry, newRegistry); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedConstantReflectionProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedConstantReflectionProvider.java index d9d162ee83a3..90ac3ccb8c5c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedConstantReflectionProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedConstantReflectionProvider.java @@ -71,9 +71,13 @@ public JavaConstant asJavaClass(ResolvedJavaType type) { @Override public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver) { + return readFieldValue(field, receiver, false); + } + + public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver, boolean readRelocatableValues) { var hField = (HostedField) field; assert checkHub(receiver) : "Receiver " + receiver + " of field " + hField + " read should not be java.lang.Class. Expecting to see DynamicHub here."; - return super.readValue(hField.getWrapped(), receiver, true); + return super.readValue(hField.getWrapped(), receiver, true, readRelocatableValues); } private boolean checkHub(JavaConstant constant) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMemoryAccessProvider.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMemoryAccessProvider.java index 03bf29215fee..586f4bb857f3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMemoryAccessProvider.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/HostedMemoryAccessProvider.java @@ -109,7 +109,7 @@ private JavaConstant doRead(JavaKind stackKind, JavaConstant base, long displace assert field.getStorageKind().getStackKind() == stackKind; JavaConstant result = hConstantReflection.readFieldValue(field, base); - if (result.getJavaKind().getStackKind() != stackKind) { + if (result == null || result.getJavaKind().getStackKind() != stackKind) { /* * For certain Word types like RelocatedPointer, the boxed value is returned by * field.readValue(). We cannot constant-fold such a field late in the lower tiers of diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/FieldValueTransformation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/FieldValueTransformation.java index 974c4ff4b774..a340ea97d322 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/FieldValueTransformation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/FieldValueTransformation.java @@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import java.util.function.Function; import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.hosted.FieldValueTransformer; @@ -36,6 +37,7 @@ import com.oracle.graal.pointsto.util.GraalAccess; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.hosted.imagelayer.ObjectToConstantFieldValueTransformer; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.ResolvedJavaField; @@ -94,9 +96,19 @@ public JavaConstant readValue(AnalysisField field, JavaConstant receiver) { private JavaConstant computeValue(AnalysisField field, JavaConstant receiver) { Object receiverValue = receiver == null ? null : GraalAccess.getOriginalSnippetReflection().asObject(Object.class, receiver); Object originalValue = fetchOriginalValue(field, receiver); - Object newValue = fieldValueTransformer.transform(receiverValue, originalValue); - checkValue(newValue, field); - JavaConstant result = GraalAccess.getOriginalSnippetReflection().forBoxed(field.getJavaKind(), newValue); + + Function constantConverter = (obj) -> { + checkValue(obj, field); + return GraalAccess.getOriginalSnippetReflection().forBoxed(field.getJavaKind(), obj); + }; + + JavaConstant result; + if (fieldValueTransformer instanceof ObjectToConstantFieldValueTransformer objectToConstantFieldValueTransformer) { + result = objectToConstantFieldValueTransformer.transformToConstant(receiverValue, originalValue, constantConverter); + } else { + Object newValue = fieldValueTransformer.transform(receiverValue, originalValue); + result = constantConverter.apply(newValue); + } assert result.getJavaKind() == field.getJavaKind(); return result;