1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26 package jdk.internal.foreign.abi.x64.sysv;
27
28 import jdk.incubator.foreign.CSupport;
29 import jdk.incubator.foreign.GroupLayout;
30 import jdk.incubator.foreign.MemoryAddress;
31 import jdk.incubator.foreign.MemoryHandles;
32 import jdk.incubator.foreign.MemoryLayout;
33 import jdk.incubator.foreign.MemorySegment;
34 import jdk.internal.foreign.NativeMemorySegmentImpl;
35 import jdk.internal.foreign.Utils;
36 import jdk.internal.foreign.abi.SharedUtils;
37 import jdk.internal.misc.Unsafe;
38
39 import java.lang.invoke.VarHandle;
40 import java.lang.ref.Cleaner;
41 import java.nio.ByteOrder;
42 import java.util.ArrayList;
43 import java.util.List;
44
45 import static jdk.incubator.foreign.CSupport.SysV;
46 import static jdk.incubator.foreign.CSupport.VaList;
47 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
48 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
49 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
50 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
51
52 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists"
53 public class SysVVaList implements VaList {
54 private static final Unsafe U = Unsafe.getUnsafe();
55
56 static final Class<?> CARRIER = MemoryAddress.class;
57
58 // struct typedef __va_list_tag __va_list_tag {
59 // unsigned int gp_offset; /* 0 4 */
60 // unsigned int fp_offset; /* 4 4 */
61 // void * overflow_arg_area; /* 8 8 */
62 // void * reg_save_area; /* 16 8 */
63 //
64 // /* size: 24, cachelines: 1, members: 4 */
65 // /* last cacheline: 24 bytes */
66 // };
67 static final GroupLayout LAYOUT = MemoryLayout.ofStruct(
68 SysV.C_INT.withName("gp_offset"),
69 SysV.C_INT.withName("fp_offset"),
70 SysV.C_POINTER.withName("overflow_arg_area"),
71 SysV.C_POINTER.withName("reg_save_area")
72 ).withName("__va_list_tag");
73
74 private static final MemoryLayout GP_REG = MemoryLayout.ofValueBits(64, ByteOrder.nativeOrder());
75 private static final MemoryLayout FP_REG = MemoryLayout.ofValueBits(128, ByteOrder.nativeOrder());
76
77 private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.ofStruct(
78 GP_REG.withName("%rdi"),
79 GP_REG.withName("%rsi"),
80 GP_REG.withName("%rdx"),
81 GP_REG.withName("%rcx"),
82 GP_REG.withName("%r8"),
83 GP_REG.withName("%r9"),
84 FP_REG.withName("%xmm0"),
85 FP_REG.withName("%xmm1"),
86 FP_REG.withName("%xmm2"),
87 FP_REG.withName("%xmm3"),
88 FP_REG.withName("%xmm4"),
89 FP_REG.withName("%xmm5"),
90 FP_REG.withName("%xmm6"),
91 FP_REG.withName("%xmm7")
92 // specification and implementation differ as to whether the following are part of a reg save area
93 // Let's go with the implementation, since then it actually works :)
94 // FP_REG.withName("%xmm8"),
95 // FP_REG.withName("%xmm9"),
96 // FP_REG.withName("%xmm10"),
97 // FP_REG.withName("%xmm11"),
98 // FP_REG.withName("%xmm12"),
99 // FP_REG.withName("%xmm13"),
100 // FP_REG.withName("%xmm14"),
101 // FP_REG.withName("%xmm15")
102 );
103
104 private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0"));
105
106 private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
107 private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
108
109 private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used
110 private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs
111
112 private static final VarHandle VH_fp_offset = LAYOUT.varHandle(int.class, groupElement("fp_offset"));
113 private static final VarHandle VH_gp_offset = LAYOUT.varHandle(int.class, groupElement("gp_offset"));
114 private static final VarHandle VH_overflow_arg_area
115 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("overflow_arg_area")));
116 private static final VarHandle VH_reg_save_area
117 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("reg_save_area")));
118
119 private static final Cleaner cleaner = Cleaner.create();
120 private static final CSupport.VaList EMPTY = new SharedUtils.EmptyVaList(emptyListAddress());
121
122 private final MemorySegment segment;
123 private final List<MemorySegment> slices = new ArrayList<>();
124 private final MemorySegment regSaveArea;
125
126 SysVVaList(MemorySegment segment) {
127 this.segment = segment;
128 regSaveArea = regSaveArea();
129 slices.add(regSaveArea);
130 }
131
132 private static MemoryAddress emptyListAddress() {
133 long ptr = U.allocateMemory(LAYOUT.byteSize());
134 MemorySegment ms = NativeMemorySegmentImpl.makeNativeSegmentUnchecked(
135 MemoryAddress.ofLong(ptr), LAYOUT.byteSize(), null, () -> U.freeMemory(ptr), null);
136 cleaner.register(SysVVaList.class, ms::close);
137 MemoryAddress base = ms.baseAddress();
138 VH_gp_offset.set(base, MAX_GP_OFFSET);
139 VH_fp_offset.set(base, MAX_FP_OFFSET);
140 VH_overflow_arg_area.set(base, MemoryAddress.NULL);
141 VH_reg_save_area.set(base, MemoryAddress.NULL);
142 return ms.withAccessModes(0).baseAddress();
143 }
144
145 public static CSupport.VaList empty() {
146 return EMPTY;
147 }
148
149 private int currentGPOffset() {
150 return (int) VH_gp_offset.get(segment.baseAddress());
151 }
152
153 private void currentGPOffset(int i) {
154 VH_gp_offset.set(segment.baseAddress(), i);
155 }
156
157 private int currentFPOffset() {
158 return (int) VH_fp_offset.get(segment.baseAddress());
159 }
160
161 private void currentFPOffset(int i) {
162 VH_fp_offset.set(segment.baseAddress(), i);
163 }
164
165 private MemoryAddress stackPtr() {
166 return (MemoryAddress) VH_overflow_arg_area.get(segment.baseAddress());
167 }
168
169 private void stackPtr(MemoryAddress ptr) {
170 VH_overflow_arg_area.set(segment.baseAddress(), ptr);
171 }
172
173 private MemorySegment regSaveArea() {
174 return MemorySegment.ofNativeRestricted((MemoryAddress) VH_reg_save_area.get(segment.baseAddress()),
175 LAYOUT_REG_SAVE_AREA.byteSize(), segment.ownerThread(), null, null);
176 }
177
178 private void preAlignStack(MemoryLayout layout) {
179 if (layout.byteAlignment() > 8) {
180 stackPtr(Utils.alignUp(stackPtr(), 16));
181 }
182 }
183
184 private void postAlignStack(MemoryLayout layout) {
185 stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
186 }
187
188 @Override
189 public int vargAsInt(MemoryLayout layout) {
190 return (int) read(int.class, layout);
191 }
192
193 @Override
194 public long vargAsLong(MemoryLayout layout) {
195 return (long) read(long.class, layout);
196 }
197
198 @Override
199 public double vargAsDouble(MemoryLayout layout) {
200 return (double) read(double.class, layout);
201 }
202
203 @Override
204 public MemoryAddress vargAsAddress(MemoryLayout layout) {
205 return (MemoryAddress) read(MemoryAddress.class, layout);
206 }
207
208 @Override
209 public MemorySegment vargAsSegment(MemoryLayout layout) {
210 return (MemorySegment) read(MemorySegment.class, layout);
211 }
212
213 private Object read(Class<?> carrier, MemoryLayout layout) {
214 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
215 TypeClass typeClass = TypeClass.classifyLayout(layout);
216 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
217 preAlignStack(layout);
218 return switch (typeClass.kind()) {
219 case STRUCT -> {
220 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
221 segment.ownerThread(), null, null)) {
222 MemorySegment seg = MemorySegment.allocateNative(layout);
223 seg.copyFrom(slice);
224 postAlignStack(layout);
225 yield seg;
226 }
227 }
228 case POINTER, INTEGER, FLOAT -> {
229 VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
230 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
231 segment.ownerThread(), null, null)) {
232 Object res = reader.get(slice.baseAddress());
233 postAlignStack(layout);
234 yield res;
235 }
236 }
237 };
238 } else {
239 return switch (typeClass.kind()) {
240 case STRUCT -> {
241 MemorySegment value = MemorySegment.allocateNative(layout);
242 int classIdx = 0;
243 long offset = 0;
244 while (offset < layout.byteSize()) {
245 final long copy = Math.min(layout.byteSize() - offset, 8);
246 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
247 MemorySegment slice = value.asSlice(offset, copy);
248 if (isSSE) {
249 slice.copyFrom(regSaveArea.asSlice(currentFPOffset(), copy));
250 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
251 } else {
252 slice.copyFrom(regSaveArea.asSlice(currentGPOffset(), copy));
253 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
254 }
255 offset += copy;
256 }
257 yield value;
258 }
259 case POINTER, INTEGER -> {
260 VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
261 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentGPOffset()));
262 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
263 yield res;
264 }
265 case FLOAT -> {
266 VarHandle reader = layout.varHandle(carrier);
267 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentFPOffset()));
268 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
269 yield res;
270 }
271 };
272 }
273 }
274
275 @Override
276 public void skip(MemoryLayout... layouts) {
277 for (MemoryLayout layout : layouts) {
278 TypeClass typeClass = TypeClass.classifyLayout(layout);
279 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
280 preAlignStack(layout);
281 postAlignStack(layout);
282 } else {
283 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE));
284 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE));
285 }
286 }
287 }
288
289 static SysVVaList.Builder builder() {
290 return new SysVVaList.Builder();
291 }
292
293 public static VaList ofAddress(MemoryAddress ma) {
294 return new SysVVaList(MemorySegment.ofNativeRestricted(ma, LAYOUT.byteSize(), Thread.currentThread(), null, null));
295 }
296
297 @Override
298 public boolean isAlive() {
299 return segment.isAlive();
300 }
301
302 @Override
303 public void close() {
304 segment.close();
305 slices.forEach(MemorySegment::close);
306 }
307
308 @Override
309 public VaList copy() {
310 MemorySegment copy = MemorySegment.allocateNative(LAYOUT.byteSize());
311 copy.copyFrom(segment);
312 return new SysVVaList(copy);
313 }
314
315 @Override
316 public MemoryAddress address() {
317 return segment.baseAddress();
318 }
319
320 private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) {
321 return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE
322 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE;
323 }
324
325 @Override
326 public String toString() {
327 return "SysVVaList{"
328 + "gp_offset=" + currentGPOffset()
329 + ", fp_offset=" + currentFPOffset()
330 + ", overflow_arg_area=" + stackPtr()
331 + ", reg_save_area=" + regSaveArea()
332 + '}';
333 }
334
335 static class Builder implements CSupport.VaList.Builder {
336 private final MemorySegment reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA);
337 private long currentGPOffset = 0;
338 private long currentFPOffset = FP_OFFSET;
339 private final List<SimpleVaArg> stackArgs = new ArrayList<>();
340
341 @Override
342 public Builder vargFromInt(MemoryLayout layout, int value) {
343 return arg(int.class, layout, value);
344 }
345
346 @Override
347 public Builder vargFromLong(MemoryLayout layout, long value) {
348 return arg(long.class, layout, value);
349 }
350
351 @Override
352 public Builder vargFromDouble(MemoryLayout layout, double value) {
353 return arg(double.class, layout, value);
354 }
355
356 @Override
357 public Builder vargFromAddress(MemoryLayout layout, MemoryAddress value) {
358 return arg(MemoryAddress.class, layout, value);
359 }
360
361 @Override
362 public Builder vargFromSegment(MemoryLayout layout, MemorySegment value) {
363 return arg(MemorySegment.class, layout, value);
364 }
365
366 private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
367 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
368 TypeClass typeClass = TypeClass.classifyLayout(layout);
369 if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)) {
370 // stack it!
371 stackArgs.add(new SimpleVaArg(carrier, layout, value));
372 } else {
373 switch (typeClass.kind()) {
374 case STRUCT -> {
375 MemorySegment valueSegment = (MemorySegment) value;
376 int classIdx = 0;
377 long offset = 0;
378 while (offset < layout.byteSize()) {
379 final long copy = Math.min(layout.byteSize() - offset, 8);
380 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
381 MemorySegment slice = valueSegment.asSlice(offset, copy);
382 if (isSSE) {
383 reg_save_area.asSlice(currentFPOffset, copy).copyFrom(slice);
384 currentFPOffset += FP_SLOT_SIZE;
385 } else {
386 reg_save_area.asSlice(currentGPOffset, copy).copyFrom(slice);
387 currentGPOffset += GP_SLOT_SIZE;
388 }
389 offset += copy;
390 }
391 }
392 case POINTER, INTEGER -> {
393 VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
394 writer.set(reg_save_area.baseAddress().addOffset(currentGPOffset), value);
395 currentGPOffset += GP_SLOT_SIZE;
396 }
397 case FLOAT -> {
398 VarHandle writer = layout.varHandle(carrier);
399 writer.set(reg_save_area.baseAddress().addOffset(currentFPOffset), value);
400 currentFPOffset += FP_SLOT_SIZE;
401 }
402 }
403 }
404 return this;
405 }
406
407 private boolean isEmpty() {
408 return currentGPOffset == 0 && currentFPOffset == FP_OFFSET && stackArgs.isEmpty();
409 }
410
411 public VaList build() {
412 if (isEmpty()) {
413 return EMPTY;
414 }
415
416 MemorySegment vaListSegment = MemorySegment.allocateNative(LAYOUT.byteSize());
417 SysVVaList res = new SysVVaList(vaListSegment);
418 MemoryAddress stackArgsPtr = MemoryAddress.NULL;
419 if (!stackArgs.isEmpty()) {
420 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum);
421 MemorySegment stackArgsSegment = MemorySegment.allocateNative(stackArgsSize, 16);
422 MemoryAddress maOverflowArgArea = stackArgsSegment.baseAddress();
423 for (SimpleVaArg arg : stackArgs) {
424 if (arg.layout.byteSize() > 8) {
425 maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize()));
426 }
427 VarHandle writer = arg.varHandle();
428 writer.set(maOverflowArgArea, arg.value);
429 maOverflowArgArea = maOverflowArgArea.addOffset(arg.layout.byteSize());
430 }
431 stackArgsPtr = stackArgsSegment.baseAddress();
432 res.slices.add(stackArgsSegment);
433 }
434
435 MemoryAddress vaListAddr = vaListSegment.baseAddress();
436 VH_fp_offset.set(vaListAddr, (int) FP_OFFSET);
437 VH_overflow_arg_area.set(vaListAddr, stackArgsPtr);
438 VH_reg_save_area.set(vaListAddr, reg_save_area.baseAddress());
439 res.slices.add(reg_save_area);
440 assert reg_save_area.ownerThread() == vaListSegment.ownerThread();
441 return res;
442 }
443 }
444 }