1 /* 2 * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package jdk.internal.foreign.abi; 24 25 import jdk.incubator.foreign.MemoryAddress; 26 import jdk.incubator.foreign.MemoryHandles; 27 import jdk.incubator.foreign.MemorySegment; 28 import jdk.incubator.foreign.NativeAllocationScope; 29 import jdk.internal.access.JavaLangInvokeAccess; 30 import jdk.internal.access.SharedSecrets; 31 import jdk.internal.foreign.MemoryAddressImpl; 32 import jdk.internal.foreign.Utils; 33 34 import java.lang.invoke.MethodHandle; 35 import java.lang.invoke.MethodHandles; 36 import java.lang.invoke.MethodType; 37 import java.lang.invoke.VarHandle; 38 import java.nio.ByteOrder; 39 import java.util.ArrayList; 40 import java.util.Arrays; 41 import java.util.List; 42 import java.util.Map; 43 import java.util.concurrent.ConcurrentHashMap; 44 import java.util.stream.Collectors; 45 import java.util.stream.IntStream; 46 47 import static java.lang.invoke.MethodHandles.collectArguments; 48 import static java.lang.invoke.MethodHandles.dropArguments; 49 import static java.lang.invoke.MethodHandles.empty; 50 import static java.lang.invoke.MethodHandles.filterArguments; 51 import static java.lang.invoke.MethodHandles.identity; 52 import static java.lang.invoke.MethodHandles.insertArguments; 53 import static java.lang.invoke.MethodHandles.permuteArguments; 54 import static java.lang.invoke.MethodHandles.tryFinally; 55 import static java.lang.invoke.MethodType.methodType; 56 import static sun.security.action.GetBooleanAction.privilegedGetProperty; 57 58 /** 59 * This class implements native call invocation through a so called 'universal adapter'. A universal adapter takes 60 * an array of longs together with a call 'recipe', which is used to move the arguments in the right places as 61 * expected by the system ABI. 62 */ 63 public class ProgrammableInvoker { 64 private static final boolean DEBUG = 65 privilegedGetProperty("jdk.internal.foreign.ProgrammableInvoker.DEBUG"); 66 private static final boolean NO_SPEC = 67 privilegedGetProperty("jdk.internal.foreign.ProgrammableInvoker.NO_SPEC"); 68 69 private static final VarHandle VH_LONG = MemoryHandles.varHandle(long.class, ByteOrder.nativeOrder()); 70 71 private static final MethodHandle MH_INVOKE_MOVES; 72 private static final MethodHandle MH_INVOKE_INTERP_BINDINGS; 73 74 private static final MethodHandle MH_UNBOX_ADDRESS; 75 private static final MethodHandle MH_BOX_ADDRESS; 76 private static final MethodHandle MH_BASE_ADDRESS; 77 private static final MethodHandle MH_COPY_BUFFER; 78 private static final MethodHandle MH_MAKE_ALLOCATOR; 79 private static final MethodHandle MH_CLOSE_ALLOCATOR; 80 private static final MethodHandle MH_ALLOCATE_BUFFER; 81 82 private static final Map<ABIDescriptor, Long> adapterStubs = new ConcurrentHashMap<>(); 83 84 static { 85 try { 86 MethodHandles.Lookup lookup = MethodHandles.lookup(); 87 MH_INVOKE_MOVES = lookup.findVirtual(ProgrammableInvoker.class, "invokeMoves", 88 methodType(Object.class, Object[].class, Binding.Move[].class, Binding.Move[].class)); 89 MH_INVOKE_INTERP_BINDINGS = lookup.findVirtual(ProgrammableInvoker.class, "invokeInterpBindings", 90 methodType(Object.class, Object[].class, MethodHandle.class, Map.class, Map.class)); 91 MH_UNBOX_ADDRESS = lookup.findStatic(ProgrammableInvoker.class, "toRawLongValue", 92 methodType(long.class, MemoryAddress.class)); 93 MH_BOX_ADDRESS = lookup.findStatic(ProgrammableInvoker.class, "ofLong", 94 methodType(MemoryAddress.class, long.class)); 95 MH_BASE_ADDRESS = lookup.findVirtual(MemorySegment.class, "baseAddress", 96 methodType(MemoryAddress.class)); 97 MH_COPY_BUFFER = lookup.findStatic(ProgrammableInvoker.class, "copyBuffer", 98 methodType(MemorySegment.class, MemorySegment.class, long.class, long.class, NativeAllocationScope.class)); 99 MH_MAKE_ALLOCATOR = lookup.findStatic(NativeAllocationScope.class, "boundedScope", 100 methodType(NativeAllocationScope.class, long.class)); 101 MH_CLOSE_ALLOCATOR = lookup.findVirtual(NativeAllocationScope.class, "close", 102 methodType(void.class)); 103 MH_ALLOCATE_BUFFER = lookup.findStatic(MemorySegment.class, "allocateNative", 104 methodType(MemorySegment.class, long.class, long.class)); 105 } catch (ReflectiveOperationException e) { 106 throw new RuntimeException(e); 107 } 108 } 109 110 private final ABIDescriptor abi; 111 private final BufferLayout layout; 112 private final long stackArgsBytes; 113 114 private final CallingSequence callingSequence; 115 116 private final MemoryAddress addr; 117 private final long stubAddress; 118 119 private final long bufferCopySize; 120 121 public ProgrammableInvoker(ABIDescriptor abi, MemoryAddress addr, CallingSequence callingSequence) { 122 this.abi = abi; 123 this.layout = BufferLayout.of(abi); 124 this.stubAddress = adapterStubs.computeIfAbsent(abi, key -> generateAdapter(key, layout)); 125 126 this.addr = addr; 127 this.callingSequence = callingSequence; 128 129 this.stackArgsBytes = callingSequence.argMoveBindings() 130 .map(Binding.Move::storage) 131 .filter(s -> abi.arch.isStackType(s.type())) 132 .count() 133 * abi.arch.typeSize(abi.arch.stackType()); 134 135 this.bufferCopySize = bufferCopySize(callingSequence); 136 } 137 138 private static long bufferCopySize(CallingSequence callingSequence) { 139 // FIXME: > 16 bytes alignment might need extra space since the 140 // starting address of the allocator might be un-aligned. 141 long size = 0; 142 for (int i = 0; i < callingSequence.argumentCount(); i++) { 143 List<Binding> bindings = callingSequence.argumentBindings(i); 144 for (Binding b : bindings) { 145 if (b instanceof Binding.Copy) { 146 Binding.Copy c = (Binding.Copy) b; 147 size = Utils.alignUp(size, c.alignment()); 148 size += c.size(); 149 } 150 } 151 } 152 return size; 153 } 154 155 public MethodHandle getBoundMethodHandle() { 156 Binding.Move[] argMoves = callingSequence.argMoveBindings().toArray(Binding.Move[]::new); 157 Class<?>[] argMoveTypes = Arrays.stream(argMoves).map(Binding.Move::type).toArray(Class<?>[]::new); 158 159 Binding.Move[] retMoves = callingSequence.retMoveBindings().toArray(Binding.Move[]::new); 160 Class<?> returnType = retMoves.length == 0 161 ? void.class 162 : retMoves.length == 1 163 ? retMoves[0].type() 164 : Object[].class; 165 166 MethodType intrinsicType = methodType(returnType, argMoveTypes); 167 168 MethodHandle handle = insertArguments(MH_INVOKE_MOVES.bindTo(this), 1, argMoves, retMoves) 169 .asCollector(Object[].class, intrinsicType.parameterCount()) 170 .asType(intrinsicType); 171 172 if (NO_SPEC || retMoves.length > 1) { 173 Map<VMStorage, Integer> argIndexMap = indexMap(argMoves); 174 Map<VMStorage, Integer> retIndexMap = indexMap(retMoves); 175 176 handle = insertArguments(MH_INVOKE_INTERP_BINDINGS.bindTo(this), 1, handle, argIndexMap, retIndexMap); 177 handle = handle.asCollector(Object[].class, callingSequence.methodType().parameterCount()) 178 .asType(callingSequence.methodType()); 179 } else { 180 handle = specialize(handle); 181 } 182 183 return handle; 184 } 185 186 private MethodHandle specialize(MethodHandle intrinsicHandle) { 187 MethodType type = callingSequence.methodType(); 188 MethodType intrinsicType = intrinsicHandle.type(); 189 190 int insertPos = -1; 191 if (bufferCopySize > 0) { 192 intrinsicHandle = dropArguments(intrinsicHandle, 0, NativeAllocationScope.class); 193 insertPos++; 194 } 195 for (int i = 0; i < type.parameterCount(); i++) { 196 List<Binding> bindings = callingSequence.argumentBindings(i); 197 insertPos += bindings.stream().filter(Binding.Move.class::isInstance).count() + 1; 198 // We interpret the bindings in reverse since we have to construct a MethodHandle from the bottom up 199 for (int j = bindings.size() - 1; j >= 0; j--) { 200 Binding binding = bindings.get(j); 201 switch (binding.tag()) { 202 case MOVE -> insertPos--; // handled by fallback 203 case DUP -> 204 intrinsicHandle = mergeArguments(intrinsicHandle, insertPos, insertPos + 1); 205 case CONVERT_ADDRESS -> 206 intrinsicHandle = filterArguments(intrinsicHandle, insertPos, MH_UNBOX_ADDRESS); 207 case BASE_ADDRESS -> 208 intrinsicHandle = filterArguments(intrinsicHandle, insertPos, MH_BASE_ADDRESS); 209 case DEREFERENCE -> { 210 Binding.Dereference deref = (Binding.Dereference) binding; 211 MethodHandle filter = filterArguments( 212 deref.varHandle() 213 .toMethodHandle(VarHandle.AccessMode.GET) 214 .asType(methodType(deref.type(), MemoryAddress.class)), 0, MH_BASE_ADDRESS); 215 intrinsicHandle = filterArguments(intrinsicHandle, insertPos, filter); 216 } 217 case COPY_BUFFER -> { 218 Binding.Copy copy = (Binding.Copy) binding; 219 MethodHandle filter = insertArguments(MH_COPY_BUFFER, 1, copy.size(), copy.alignment()); 220 intrinsicHandle = collectArguments(intrinsicHandle, insertPos, filter); 221 intrinsicHandle = mergeArguments(intrinsicHandle, 0, insertPos + 1); 222 } 223 default -> throw new IllegalArgumentException("Illegal tag: " + binding.tag()); 224 } 225 } 226 } 227 228 if (type.returnType() != void.class) { 229 MethodHandle returnFilter = identity(type.returnType()); 230 List<Binding> bindings = callingSequence.returnBindings(); 231 for (int j = bindings.size() - 1; j >= 0; j--) { 232 Binding binding = bindings.get(j); 233 switch (binding.tag()) { 234 case MOVE -> { /* handled by fallback */ } 235 case CONVERT_ADDRESS -> 236 returnFilter = filterArguments(returnFilter, 0, MH_BOX_ADDRESS); 237 case DEREFERENCE -> { 238 Binding.Dereference deref = (Binding.Dereference) binding; 239 MethodHandle setter = deref.varHandle().toMethodHandle(VarHandle.AccessMode.SET); 240 setter = filterArguments( 241 setter.asType(methodType(void.class, MemoryAddress.class, deref.type())), 242 0, MH_BASE_ADDRESS); 243 returnFilter = collectArguments(returnFilter, returnFilter.type().parameterCount(), setter); 244 } 245 case DUP -> 246 // FIXME assumes shape like: (MS, ..., MS, T) R, is that good enough? 247 returnFilter = mergeArguments(returnFilter, 0, returnFilter.type().parameterCount() - 2); 248 case ALLOC_BUFFER -> { 249 Binding.Allocate alloc = (Binding.Allocate) binding; 250 returnFilter = collectArguments(returnFilter, 0, 251 insertArguments(MH_ALLOCATE_BUFFER, 0, alloc.size(), alloc.alignment())); 252 } 253 default -> 254 throw new IllegalArgumentException("Illegal tag: " + binding.tag()); 255 } 256 } 257 258 intrinsicHandle = MethodHandles.filterReturnValue(intrinsicHandle, returnFilter); 259 } 260 261 if (bufferCopySize > 0) { 262 MethodHandle closer = intrinsicType.returnType() == void.class 263 // (Throwable, NativeAllocationScope) -> void 264 ? collectArguments(empty(methodType(void.class, Throwable.class)), 1, MH_CLOSE_ALLOCATOR) 265 // (Throwable, V, NativeAllocationScope) -> V 266 : collectArguments(dropArguments(identity(intrinsicHandle.type().returnType()), 0, Throwable.class), 267 2, MH_CLOSE_ALLOCATOR); 268 intrinsicHandle = tryFinally(intrinsicHandle, closer); 269 intrinsicHandle = collectArguments(intrinsicHandle, 0, insertArguments(MH_MAKE_ALLOCATOR, 0, bufferCopySize)); 270 } 271 return intrinsicHandle; 272 } 273 274 private static MethodHandle mergeArguments(MethodHandle mh, int sourceIndex, int destIndex) { 275 MethodType oldType = mh.type(); 276 Class<?> sourceType = oldType.parameterType(sourceIndex); 277 Class<?> destType = oldType.parameterType(destIndex); 278 if (sourceType != destType) { 279 // TODO meet? 280 throw new IllegalArgumentException("Parameter types differ: " + sourceType + " != " + destType); 281 } 282 MethodType newType = oldType.dropParameterTypes(destIndex, destIndex + 1); 283 int[] reorder = new int[oldType.parameterCount()]; 284 assert destIndex > sourceIndex; 285 for (int i = 0, index = 0; i < reorder.length; i++) { 286 if (i != destIndex) { 287 reorder[i] = index++; 288 } else { 289 reorder[i] = sourceIndex; 290 } 291 } 292 return permuteArguments(mh, newType, reorder); 293 } 294 295 private static MemorySegment copyBuffer(MemorySegment operand, long size, long alignment, 296 NativeAllocationScope allocator) { 297 assert operand.byteSize() == size : "operand size mismatch"; 298 MemorySegment copy = allocator.allocate(size, alignment).segment(); 299 copy.copyFrom(operand.asSlice(0, size)); 300 return copy; 301 } 302 303 private static long toRawLongValue(MemoryAddress address) { 304 return address.toRawLongValue(); // Workaround for JDK-8239083 305 } 306 307 private static MemoryAddress ofLong(long address) { 308 return MemoryAddress.ofLong(address); // Workaround for JDK-8239083 309 } 310 311 private Map<VMStorage, Integer> indexMap(Binding.Move[] moves) { 312 return IntStream.range(0, moves.length) 313 .boxed() 314 .collect(Collectors.toMap(i -> moves[i].storage(), i -> i)); 315 } 316 317 /** 318 * Does a native invocation by moving primitive values from the arg array into an intermediate buffer 319 * and calling the assembly stub that forwards arguments from the buffer to the target function 320 * 321 * @param args an array of primitive values to be copied in to the buffer 322 * @param argBindings Binding.Move values describing how arguments should be copied 323 * @param returnBindings Binding.Move values describing how return values should be copied 324 * @return null, a single primitive value, or an Object[] of primitive values 325 */ 326 Object invokeMoves(Object[] args, Binding.Move[] argBindings, Binding.Move[] returnBindings) { 327 MemorySegment stackArgsSeg = null; 328 try (MemorySegment argBuffer = MemorySegment.allocateNative(layout.size, 64)) { 329 MemoryAddress argsPtr = argBuffer.baseAddress(); 330 MemoryAddress stackArgs; 331 if (stackArgsBytes > 0) { 332 stackArgsSeg = MemorySegment.allocateNative(stackArgsBytes, 8); 333 stackArgs = stackArgsSeg.baseAddress(); 334 } else { 335 stackArgs = MemoryAddressImpl.NULL; 336 } 337 338 VH_LONG.set(argsPtr.addOffset(layout.arguments_next_pc), addr.toRawLongValue()); 339 VH_LONG.set(argsPtr.addOffset(layout.stack_args_bytes), stackArgsBytes); 340 VH_LONG.set(argsPtr.addOffset(layout.stack_args), stackArgs.toRawLongValue()); 341 342 for (int i = 0; i < argBindings.length; i++) { 343 Binding.Move binding = argBindings[i]; 344 VMStorage storage = binding.storage(); 345 MemoryAddress ptr = abi.arch.isStackType(storage.type()) 346 ? stackArgs.addOffset(storage.index() * abi.arch.typeSize(abi.arch.stackType())) 347 : argsPtr.addOffset(layout.argOffset(storage)); 348 SharedUtils.writeOverSized(ptr, binding.type(), args[i]); 349 } 350 351 if (DEBUG) { 352 System.err.println("Buffer state before:"); 353 layout.dump(abi.arch, argsPtr, System.err); 354 } 355 356 invokeNative(stubAddress, argsPtr.toRawLongValue()); 357 358 if (DEBUG) { 359 System.err.println("Buffer state after:"); 360 layout.dump(abi.arch, argsPtr, System.err); 361 } 362 363 if (returnBindings.length == 0) { 364 return null; 365 } else if (returnBindings.length == 1) { 366 Binding.Move move = returnBindings[0]; 367 VMStorage storage = move.storage(); 368 return SharedUtils.read(argsPtr.addOffset(layout.retOffset(storage)), move.type()); 369 } else { // length > 1 370 Object[] returns = new Object[returnBindings.length]; 371 for (int i = 0; i < returnBindings.length; i++) { 372 Binding.Move move = returnBindings[i]; 373 VMStorage storage = move.storage(); 374 returns[i] = SharedUtils.read(argsPtr.addOffset(layout.retOffset(storage)), move.type()); 375 } 376 return returns; 377 } 378 } finally { 379 if (stackArgsSeg != null) { 380 stackArgsSeg.close(); 381 } 382 } 383 } 384 385 Object invokeInterpBindings(Object[] args, MethodHandle leaf, 386 Map<VMStorage, Integer> argIndexMap, 387 Map<VMStorage, Integer> retIndexMap) throws Throwable { 388 List<MemorySegment> tempBuffers = new ArrayList<>(); 389 try { 390 // do argument processing, get Object[] as result 391 Object[] moves = new Object[leaf.type().parameterCount()]; 392 for (int i = 0; i < args.length; i++) { 393 Object arg = args[i]; 394 BindingInterpreter.unbox(arg, callingSequence.argumentBindings(i), 395 (storage, type, value) -> { 396 moves[argIndexMap.get(storage)] = value; 397 }, tempBuffers); 398 } 399 400 // call leaf 401 Object o = leaf.invokeWithArguments(moves); 402 403 // return value processing 404 if (o == null) { 405 return null; 406 } else if (o instanceof Object[]) { 407 Object[] oArr = (Object[]) o; 408 return BindingInterpreter.box(callingSequence.returnBindings(), 409 (storage, type) -> oArr[retIndexMap.get(storage)]); 410 } else { 411 return BindingInterpreter.box(callingSequence.returnBindings(), (storage, type) -> o); 412 } 413 } finally { 414 tempBuffers.forEach(MemorySegment::close); 415 } 416 } 417 418 //natives 419 420 static native void invokeNative(long adapterStub, long buff); 421 static native long generateAdapter(ABIDescriptor abi, BufferLayout layout); 422 423 private static native void registerNatives(); 424 static { 425 registerNatives(); 426 } 427 } 428