1 /* 2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 package jdk.internal.foreign.abi.x64.sysv; 27 28 import jdk.incubator.foreign.CSupport; 29 import jdk.incubator.foreign.GroupLayout; 30 import jdk.incubator.foreign.MemoryAddress; 31 import jdk.incubator.foreign.MemoryHandles; 32 import jdk.incubator.foreign.MemoryLayout; 33 import jdk.incubator.foreign.MemorySegment; 34 import jdk.internal.foreign.Utils; 35 import jdk.internal.foreign.abi.SharedUtils; 36 37 import java.lang.invoke.VarHandle; 38 import java.nio.ByteOrder; 39 import java.util.ArrayList; 40 import java.util.List; 41 42 import static jdk.incubator.foreign.CSupport.SysV; 43 import static jdk.incubator.foreign.CSupport.VaList; 44 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement; 45 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg; 46 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType; 47 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress; 48 49 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists" 50 public class SysVVaList implements VaList { 51 static final Class<?> CARRIER = MemoryAddress.class; 52 53 // struct typedef __va_list_tag __va_list_tag { 54 // unsigned int gp_offset; /* 0 4 */ 55 // unsigned int fp_offset; /* 4 4 */ 56 // void * overflow_arg_area; /* 8 8 */ 57 // void * reg_save_area; /* 16 8 */ 58 // 59 // /* size: 24, cachelines: 1, members: 4 */ 60 // /* last cacheline: 24 bytes */ 61 // }; 62 static final GroupLayout LAYOUT = MemoryLayout.ofStruct( 63 SysV.C_INT.withName("gp_offset"), 64 SysV.C_INT.withName("fp_offset"), 65 SysV.C_POINTER.withName("overflow_arg_area"), 66 SysV.C_POINTER.withName("reg_save_area") 67 ).withName("__va_list_tag"); 68 69 private static final MemoryLayout GP_REG = MemoryLayout.ofValueBits(64, ByteOrder.nativeOrder()); 70 private static final MemoryLayout FP_REG = MemoryLayout.ofValueBits(128, ByteOrder.nativeOrder()); 71 72 private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.ofStruct( 73 GP_REG.withName("%rdi"), 74 GP_REG.withName("%rsi"), 75 GP_REG.withName("%rdx"), 76 GP_REG.withName("%rcx"), 77 GP_REG.withName("%r8"), 78 GP_REG.withName("%r9"), 79 FP_REG.withName("%xmm0"), 80 FP_REG.withName("%xmm1"), 81 FP_REG.withName("%xmm2"), 82 FP_REG.withName("%xmm3"), 83 FP_REG.withName("%xmm4"), 84 FP_REG.withName("%xmm5"), 85 FP_REG.withName("%xmm6"), 86 FP_REG.withName("%xmm7") 87 // specification and implementation differ as to whether the following are part of a reg save area 88 // Let's go with the implementation, since then it actually works :) 89 // FP_REG.withName("%xmm8"), 90 // FP_REG.withName("%xmm9"), 91 // FP_REG.withName("%xmm10"), 92 // FP_REG.withName("%xmm11"), 93 // FP_REG.withName("%xmm12"), 94 // FP_REG.withName("%xmm13"), 95 // FP_REG.withName("%xmm14"), 96 // FP_REG.withName("%xmm15") 97 ); 98 99 private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0")); 100 101 private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize(); 102 private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize(); 103 104 private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used 105 private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs 106 107 private static final VarHandle VH_fp_offset = LAYOUT.varHandle(int.class, groupElement("fp_offset")); 108 private static final VarHandle VH_gp_offset = LAYOUT.varHandle(int.class, groupElement("gp_offset")); 109 private static final VarHandle VH_overflow_arg_area 110 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("overflow_arg_area"))); 111 private static final VarHandle VH_reg_save_area 112 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("reg_save_area"))); 113 114 private final MemorySegment segment; 115 private final List<MemorySegment> slices = new ArrayList<>(); 116 private final MemorySegment regSaveArea; 117 118 SysVVaList(MemorySegment segment) { 119 this.segment = segment; 120 regSaveArea = regSaveArea(); 121 slices.add(regSaveArea); 122 } 123 124 private int currentGPOffset() { 125 return (int) VH_gp_offset.get(segment.baseAddress()); 126 } 127 128 private void currentGPOffset(int i) { 129 VH_gp_offset.set(segment.baseAddress(), i); 130 } 131 132 private int currentFPOffset() { 133 return (int) VH_fp_offset.get(segment.baseAddress()); 134 } 135 136 private void currentFPOffset(int i) { 137 VH_fp_offset.set(segment.baseAddress(), i); 138 } 139 140 private MemoryAddress stackPtr() { 141 return (MemoryAddress) VH_overflow_arg_area.get(segment.baseAddress()); 142 } 143 144 private void stackPtr(MemoryAddress ptr) { 145 VH_overflow_arg_area.set(segment.baseAddress(), ptr); 146 } 147 148 private MemorySegment regSaveArea() { 149 return MemorySegment.ofNativeRestricted((MemoryAddress) VH_reg_save_area.get(segment.baseAddress()), 150 LAYOUT_REG_SAVE_AREA.byteSize(), segment.ownerThread(), null, null); 151 } 152 153 private void preAlignStack(MemoryLayout layout) { 154 if (layout.byteAlignment() > 8) { 155 stackPtr(Utils.alignUp(stackPtr(), 16)); 156 } 157 } 158 159 private void postAlignStack(MemoryLayout layout) { 160 stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8)); 161 } 162 163 @Override 164 public int vargAsInt(MemoryLayout layout) { 165 return (int) read(int.class, layout); 166 } 167 168 @Override 169 public long vargAsLong(MemoryLayout layout) { 170 return (long) read(long.class, layout); 171 } 172 173 @Override 174 public double vargAsDouble(MemoryLayout layout) { 175 return (double) read(double.class, layout); 176 } 177 178 @Override 179 public MemoryAddress vargAsAddress(MemoryLayout layout) { 180 return (MemoryAddress) read(MemoryAddress.class, layout); 181 } 182 183 @Override 184 public MemorySegment vargAsSegment(MemoryLayout layout) { 185 return (MemorySegment) read(MemorySegment.class, layout); 186 } 187 188 private Object read(Class<?> carrier, MemoryLayout layout) { 189 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE); 190 TypeClass typeClass = TypeClass.classifyLayout(layout); 191 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) { 192 preAlignStack(layout); 193 return switch (typeClass.kind()) { 194 case STRUCT -> { 195 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(), 196 segment.ownerThread(), null, null)) { 197 MemorySegment seg = MemorySegment.allocateNative(layout); 198 seg.copyFrom(slice); 199 postAlignStack(layout); 200 yield seg; 201 } 202 } 203 case POINTER, INTEGER, FLOAT -> { 204 VarHandle reader = vhPrimitiveOrAddress(carrier, layout); 205 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(), 206 segment.ownerThread(), null, null)) { 207 Object res = reader.get(slice.baseAddress()); 208 postAlignStack(layout); 209 yield res; 210 } 211 } 212 }; 213 } else { 214 return switch (typeClass.kind()) { 215 case STRUCT -> { 216 MemorySegment value = MemorySegment.allocateNative(layout); 217 int classIdx = 0; 218 long offset = 0; 219 while (offset < layout.byteSize()) { 220 final long copy = Math.min(layout.byteSize() - offset, 8); 221 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE; 222 MemorySegment slice = value.asSlice(offset, copy); 223 if (isSSE) { 224 slice.copyFrom(regSaveArea.asSlice(currentFPOffset(), copy)); 225 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE); 226 } else { 227 slice.copyFrom(regSaveArea.asSlice(currentGPOffset(), copy)); 228 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE); 229 } 230 offset += copy; 231 } 232 yield value; 233 } 234 case POINTER, INTEGER -> { 235 VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout); 236 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentGPOffset())); 237 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE); 238 yield res; 239 } 240 case FLOAT -> { 241 VarHandle reader = layout.varHandle(carrier); 242 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentFPOffset())); 243 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE); 244 yield res; 245 } 246 }; 247 } 248 } 249 250 @Override 251 public void skip(MemoryLayout... layouts) { 252 for (MemoryLayout layout : layouts) { 253 TypeClass typeClass = TypeClass.classifyLayout(layout); 254 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) { 255 preAlignStack(layout); 256 postAlignStack(layout); 257 } else { 258 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE)); 259 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE)); 260 } 261 } 262 } 263 264 static SysVVaList.Builder builder() { 265 return new SysVVaList.Builder(); 266 } 267 268 public static VaList ofAddress(MemoryAddress ma) { 269 return new SysVVaList(MemorySegment.ofNativeRestricted(ma, LAYOUT.byteSize(), Thread.currentThread(), null, null)); 270 } 271 272 MemorySegment getSegment() { 273 return segment; 274 } 275 276 @Override 277 public boolean isAlive() { 278 return segment.isAlive(); 279 } 280 281 @Override 282 public void close() { 283 segment.close(); 284 slices.forEach(MemorySegment::close); 285 } 286 287 @Override 288 public VaList copy() { 289 MemorySegment copy = MemorySegment.allocateNative(LAYOUT.byteSize()); 290 copy.copyFrom(segment); 291 return new SysVVaList(copy); 292 } 293 294 @Override 295 public MemoryAddress address() { 296 return segment.baseAddress(); 297 } 298 299 private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) { 300 return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE 301 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE; 302 } 303 304 @Override 305 public String toString() { 306 return "SysVVaList{" 307 + "gp_offset=" + currentGPOffset() 308 + ", fp_offset=" + currentFPOffset() 309 + ", overflow_arg_area=" + stackPtr() 310 + ", reg_save_area=" + regSaveArea() 311 + '}'; 312 } 313 314 static class Builder implements CSupport.VaList.Builder { 315 private final MemorySegment reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA); 316 private long currentGPOffset = 0; 317 private long currentFPOffset = FP_OFFSET; 318 private final List<SimpleVaArg> stackArgs = new ArrayList<>(); 319 320 @Override 321 public Builder vargFromInt(MemoryLayout layout, int value) { 322 return arg(int.class, layout, value); 323 } 324 325 @Override 326 public Builder vargFromLong(MemoryLayout layout, long value) { 327 return arg(long.class, layout, value); 328 } 329 330 @Override 331 public Builder vargFromDouble(MemoryLayout layout, double value) { 332 return arg(double.class, layout, value); 333 } 334 335 @Override 336 public Builder vargFromAddress(MemoryLayout layout, MemoryAddress value) { 337 return arg(MemoryAddress.class, layout, value); 338 } 339 340 @Override 341 public Builder vargFromSegment(MemoryLayout layout, MemorySegment value) { 342 return arg(MemorySegment.class, layout, value); 343 } 344 345 private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) { 346 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE); 347 TypeClass typeClass = TypeClass.classifyLayout(layout); 348 if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)) { 349 // stack it! 350 stackArgs.add(new SimpleVaArg(carrier, layout, value)); 351 } else { 352 switch (typeClass.kind()) { 353 case STRUCT -> { 354 MemorySegment valueSegment = (MemorySegment) value; 355 int classIdx = 0; 356 long offset = 0; 357 while (offset < layout.byteSize()) { 358 final long copy = Math.min(layout.byteSize() - offset, 8); 359 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE; 360 MemorySegment slice = valueSegment.asSlice(offset, copy); 361 if (isSSE) { 362 reg_save_area.asSlice(currentFPOffset, copy).copyFrom(slice); 363 currentFPOffset += FP_SLOT_SIZE; 364 } else { 365 reg_save_area.asSlice(currentGPOffset, copy).copyFrom(slice); 366 currentGPOffset += GP_SLOT_SIZE; 367 } 368 offset += copy; 369 } 370 } 371 case POINTER, INTEGER -> { 372 VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout); 373 writer.set(reg_save_area.baseAddress().addOffset(currentGPOffset), value); 374 currentGPOffset += GP_SLOT_SIZE; 375 } 376 case FLOAT -> { 377 VarHandle writer = layout.varHandle(carrier); 378 writer.set(reg_save_area.baseAddress().addOffset(currentFPOffset), value); 379 currentFPOffset += FP_SLOT_SIZE; 380 } 381 } 382 } 383 return this; 384 } 385 386 public SysVVaList build() { 387 MemorySegment vaListSegment = MemorySegment.allocateNative(LAYOUT.byteSize()); 388 SysVVaList res = new SysVVaList(vaListSegment); 389 MemoryAddress stackArgsPtr = MemoryAddress.NULL; 390 if (!stackArgs.isEmpty()) { 391 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum); 392 MemorySegment stackArgsSegment = MemorySegment.allocateNative(stackArgsSize, 16); 393 MemoryAddress maOverflowArgArea = stackArgsSegment.baseAddress(); 394 for (SimpleVaArg arg : stackArgs) { 395 if (arg.layout.byteSize() > 8) { 396 maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize())); 397 } 398 VarHandle writer = arg.varHandle(); 399 writer.set(maOverflowArgArea, arg.value); 400 maOverflowArgArea = maOverflowArgArea.addOffset(arg.layout.byteSize()); 401 } 402 stackArgsPtr = stackArgsSegment.baseAddress(); 403 res.slices.add(stackArgsSegment); 404 } 405 406 MemoryAddress vaListAddr = vaListSegment.baseAddress(); 407 VH_fp_offset.set(vaListAddr, (int) FP_OFFSET); 408 VH_overflow_arg_area.set(vaListAddr, stackArgsPtr); 409 VH_reg_save_area.set(vaListAddr, reg_save_area.baseAddress()); 410 res.slices.add(reg_save_area); 411 assert reg_save_area.ownerThread() == vaListSegment.ownerThread(); 412 return res; 413 } 414 } 415 }