1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26 package jdk.internal.foreign.abi.x64.sysv;
27
28 import jdk.incubator.foreign.CSupport;
29 import jdk.incubator.foreign.GroupLayout;
30 import jdk.incubator.foreign.MemoryAddress;
31 import jdk.incubator.foreign.MemoryHandles;
32 import jdk.incubator.foreign.MemoryLayout;
33 import jdk.incubator.foreign.MemorySegment;
34 import jdk.internal.foreign.AbstractMemorySegmentImpl;
35 import jdk.internal.foreign.NativeMemorySegmentImpl;
36 import jdk.internal.foreign.Utils;
37 import jdk.internal.foreign.abi.SharedUtils;
38
39 import java.lang.invoke.VarHandle;
40 import java.lang.ref.Cleaner;
41 import java.nio.ByteOrder;
42 import java.util.ArrayList;
43 import java.util.List;
44
45 import static jdk.incubator.foreign.CSupport.SysV;
46 import static jdk.incubator.foreign.CSupport.VaList;
47 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
48 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
49 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
50 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
51
52 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists"
53 public class SysVVaList implements VaList {
54 static final Class<?> CARRIER = MemoryAddress.class;
55
56 // struct typedef __va_list_tag __va_list_tag {
57 // unsigned int gp_offset; /* 0 4 */
58 // unsigned int fp_offset; /* 4 4 */
59 // void * overflow_arg_area; /* 8 8 */
60 // void * reg_save_area; /* 16 8 */
61 //
62 // /* size: 24, cachelines: 1, members: 4 */
63 // /* last cacheline: 24 bytes */
64 // };
65 static final GroupLayout LAYOUT = MemoryLayout.ofStruct(
66 SysV.C_INT.withName("gp_offset"),
67 SysV.C_INT.withName("fp_offset"),
68 SysV.C_POINTER.withName("overflow_arg_area"),
69 SysV.C_POINTER.withName("reg_save_area")
70 ).withName("__va_list_tag");
71
72 private static final MemoryLayout GP_REG = MemoryLayout.ofValueBits(64, ByteOrder.nativeOrder());
73 private static final MemoryLayout FP_REG = MemoryLayout.ofValueBits(128, ByteOrder.nativeOrder());
74
75 private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.ofStruct(
76 GP_REG.withName("%rdi"),
77 GP_REG.withName("%rsi"),
78 GP_REG.withName("%rdx"),
79 GP_REG.withName("%rcx"),
80 GP_REG.withName("%r8"),
81 GP_REG.withName("%r9"),
82 FP_REG.withName("%xmm0"),
83 FP_REG.withName("%xmm1"),
84 FP_REG.withName("%xmm2"),
85 FP_REG.withName("%xmm3"),
86 FP_REG.withName("%xmm4"),
87 FP_REG.withName("%xmm5"),
88 FP_REG.withName("%xmm6"),
89 FP_REG.withName("%xmm7")
90 // specification and implementation differ as to whether the following are part of a reg save area
91 // Let's go with the implementation, since then it actually works :)
92 // FP_REG.withName("%xmm8"),
93 // FP_REG.withName("%xmm9"),
94 // FP_REG.withName("%xmm10"),
95 // FP_REG.withName("%xmm11"),
96 // FP_REG.withName("%xmm12"),
97 // FP_REG.withName("%xmm13"),
98 // FP_REG.withName("%xmm14"),
99 // FP_REG.withName("%xmm15")
100 );
101
102 private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0"));
103
104 private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
105 private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
106
107 private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used
108 private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs
109
110 private static final VarHandle VH_fp_offset = LAYOUT.varHandle(int.class, groupElement("fp_offset"));
111 private static final VarHandle VH_gp_offset = LAYOUT.varHandle(int.class, groupElement("gp_offset"));
112 private static final VarHandle VH_overflow_arg_area
113 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("overflow_arg_area")));
114 private static final VarHandle VH_reg_save_area
115 = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("reg_save_area")));
116
117 private static final Cleaner cleaner = Cleaner.create();
118 private static final CSupport.VaList EMPTY = new SharedUtils.EmptyVaList(emptyListAddress());
119
120 private final MemorySegment segment;
121 private final List<MemorySegment> slices = new ArrayList<>();
122 private final MemorySegment regSaveArea;
123
124 SysVVaList(MemorySegment segment) {
125 this.segment = segment;
126 regSaveArea = regSaveArea();
127 slices.add(regSaveArea);
128 }
129
130 private static MemoryAddress emptyListAddress() {
131 MemorySegment ms = MemorySegment.allocateNative(LAYOUT);
132 cleaner.register(SysVVaList.class, ms::close);
133 MemoryAddress base = ms.baseAddress();
134 VH_gp_offset.set(base, MAX_GP_OFFSET);
135 VH_fp_offset.set(base, MAX_FP_OFFSET);
136 VH_overflow_arg_area.set(base, MemoryAddress.NULL);
137 VH_reg_save_area.set(base, MemoryAddress.NULL);
138 MemorySegment unconfined = NativeMemorySegmentImpl.makeNativeSegmentUnchecked(
139 base, ms.byteSize(), null, null, null).withAccessModes(0);
140 return unconfined.baseAddress();
141 }
142
143 public static CSupport.VaList empty() {
144 return EMPTY;
145 }
146
147 private int currentGPOffset() {
148 return (int) VH_gp_offset.get(segment.baseAddress());
149 }
150
151 private void currentGPOffset(int i) {
152 VH_gp_offset.set(segment.baseAddress(), i);
153 }
154
155 private int currentFPOffset() {
156 return (int) VH_fp_offset.get(segment.baseAddress());
157 }
158
159 private void currentFPOffset(int i) {
160 VH_fp_offset.set(segment.baseAddress(), i);
161 }
162
163 private MemoryAddress stackPtr() {
164 return (MemoryAddress) VH_overflow_arg_area.get(segment.baseAddress());
165 }
166
167 private void stackPtr(MemoryAddress ptr) {
168 VH_overflow_arg_area.set(segment.baseAddress(), ptr);
169 }
170
171 private MemorySegment regSaveArea() {
172 return MemorySegment.ofNativeRestricted((MemoryAddress) VH_reg_save_area.get(segment.baseAddress()),
173 LAYOUT_REG_SAVE_AREA.byteSize(), segment.ownerThread(), null, null);
174 }
175
176 private void preAlignStack(MemoryLayout layout) {
177 if (layout.byteAlignment() > 8) {
178 stackPtr(Utils.alignUp(stackPtr(), 16));
179 }
180 }
181
182 private void postAlignStack(MemoryLayout layout) {
183 stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
184 }
185
186 @Override
187 public int vargAsInt(MemoryLayout layout) {
188 return (int) read(int.class, layout);
189 }
190
191 @Override
192 public long vargAsLong(MemoryLayout layout) {
193 return (long) read(long.class, layout);
194 }
195
196 @Override
197 public double vargAsDouble(MemoryLayout layout) {
198 return (double) read(double.class, layout);
199 }
200
201 @Override
202 public MemoryAddress vargAsAddress(MemoryLayout layout) {
203 return (MemoryAddress) read(MemoryAddress.class, layout);
204 }
205
206 @Override
207 public MemorySegment vargAsSegment(MemoryLayout layout) {
208 return (MemorySegment) read(MemorySegment.class, layout);
209 }
210
211 private Object read(Class<?> carrier, MemoryLayout layout) {
212 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
213 TypeClass typeClass = TypeClass.classifyLayout(layout);
214 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
215 preAlignStack(layout);
216 return switch (typeClass.kind()) {
217 case STRUCT -> {
218 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
219 segment.ownerThread(), null, null)) {
220 MemorySegment seg = MemorySegment.allocateNative(layout);
221 seg.copyFrom(slice);
222 postAlignStack(layout);
223 yield seg;
224 }
225 }
226 case POINTER, INTEGER, FLOAT -> {
227 VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
228 try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
229 segment.ownerThread(), null, null)) {
230 Object res = reader.get(slice.baseAddress());
231 postAlignStack(layout);
232 yield res;
233 }
234 }
235 };
236 } else {
237 return switch (typeClass.kind()) {
238 case STRUCT -> {
239 MemorySegment value = MemorySegment.allocateNative(layout);
240 int classIdx = 0;
241 long offset = 0;
242 while (offset < layout.byteSize()) {
243 final long copy = Math.min(layout.byteSize() - offset, 8);
244 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
245 MemorySegment slice = value.asSlice(offset, copy);
246 if (isSSE) {
247 slice.copyFrom(regSaveArea.asSlice(currentFPOffset(), copy));
248 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
249 } else {
250 slice.copyFrom(regSaveArea.asSlice(currentGPOffset(), copy));
251 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
252 }
253 offset += copy;
254 }
255 yield value;
256 }
257 case POINTER, INTEGER -> {
258 VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
259 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentGPOffset()));
260 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
261 yield res;
262 }
263 case FLOAT -> {
264 VarHandle reader = layout.varHandle(carrier);
265 Object res = reader.get(regSaveArea.baseAddress().addOffset(currentFPOffset()));
266 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
267 yield res;
268 }
269 };
270 }
271 }
272
273 @Override
274 public void skip(MemoryLayout... layouts) {
275 for (MemoryLayout layout : layouts) {
276 TypeClass typeClass = TypeClass.classifyLayout(layout);
277 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
278 preAlignStack(layout);
279 postAlignStack(layout);
280 } else {
281 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE));
282 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE));
283 }
284 }
285 }
286
287 static SysVVaList.Builder builder() {
288 return new SysVVaList.Builder();
289 }
290
291 public static VaList ofAddress(MemoryAddress ma) {
292 return new SysVVaList(MemorySegment.ofNativeRestricted(ma, LAYOUT.byteSize(), Thread.currentThread(), null, null));
293 }
294
295 @Override
296 public boolean isAlive() {
297 return segment.isAlive();
298 }
299
300 @Override
301 public void close() {
302 segment.close();
303 slices.forEach(MemorySegment::close);
304 }
305
306 @Override
307 public VaList copy() {
308 MemorySegment copy = MemorySegment.allocateNative(LAYOUT.byteSize());
309 copy.copyFrom(segment);
310 return new SysVVaList(copy);
311 }
312
313 @Override
314 public MemoryAddress address() {
315 return segment.baseAddress();
316 }
317
318 private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) {
319 return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE
320 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE;
321 }
322
323 @Override
324 public String toString() {
325 return "SysVVaList{"
326 + "gp_offset=" + currentGPOffset()
327 + ", fp_offset=" + currentFPOffset()
328 + ", overflow_arg_area=" + stackPtr()
329 + ", reg_save_area=" + regSaveArea()
330 + '}';
331 }
332
333 static class Builder implements CSupport.VaList.Builder {
334 private final MemorySegment reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA);
335 private long currentGPOffset = 0;
336 private long currentFPOffset = FP_OFFSET;
337 private final List<SimpleVaArg> stackArgs = new ArrayList<>();
338
339 @Override
340 public Builder vargFromInt(MemoryLayout layout, int value) {
341 return arg(int.class, layout, value);
342 }
343
344 @Override
345 public Builder vargFromLong(MemoryLayout layout, long value) {
346 return arg(long.class, layout, value);
347 }
348
349 @Override
350 public Builder vargFromDouble(MemoryLayout layout, double value) {
351 return arg(double.class, layout, value);
352 }
353
354 @Override
355 public Builder vargFromAddress(MemoryLayout layout, MemoryAddress value) {
356 return arg(MemoryAddress.class, layout, value);
357 }
358
359 @Override
360 public Builder vargFromSegment(MemoryLayout layout, MemorySegment value) {
361 return arg(MemorySegment.class, layout, value);
362 }
363
364 private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
365 checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
366 TypeClass typeClass = TypeClass.classifyLayout(layout);
367 if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)) {
368 // stack it!
369 stackArgs.add(new SimpleVaArg(carrier, layout, value));
370 } else {
371 switch (typeClass.kind()) {
372 case STRUCT -> {
373 MemorySegment valueSegment = (MemorySegment) value;
374 int classIdx = 0;
375 long offset = 0;
376 while (offset < layout.byteSize()) {
377 final long copy = Math.min(layout.byteSize() - offset, 8);
378 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
379 MemorySegment slice = valueSegment.asSlice(offset, copy);
380 if (isSSE) {
381 reg_save_area.asSlice(currentFPOffset, copy).copyFrom(slice);
382 currentFPOffset += FP_SLOT_SIZE;
383 } else {
384 reg_save_area.asSlice(currentGPOffset, copy).copyFrom(slice);
385 currentGPOffset += GP_SLOT_SIZE;
386 }
387 offset += copy;
388 }
389 }
390 case POINTER, INTEGER -> {
391 VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
392 writer.set(reg_save_area.baseAddress().addOffset(currentGPOffset), value);
393 currentGPOffset += GP_SLOT_SIZE;
394 }
395 case FLOAT -> {
396 VarHandle writer = layout.varHandle(carrier);
397 writer.set(reg_save_area.baseAddress().addOffset(currentFPOffset), value);
398 currentFPOffset += FP_SLOT_SIZE;
399 }
400 }
401 }
402 return this;
403 }
404
405 private boolean isEmpty() {
406 return currentGPOffset == 0 && currentFPOffset == FP_OFFSET && stackArgs.isEmpty();
407 }
408
409 public VaList build() {
410 if (isEmpty()) {
411 return EMPTY;
412 }
413
414 MemorySegment vaListSegment = MemorySegment.allocateNative(LAYOUT.byteSize());
415 SysVVaList res = new SysVVaList(vaListSegment);
416 MemoryAddress stackArgsPtr = MemoryAddress.NULL;
417 if (!stackArgs.isEmpty()) {
418 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum);
419 MemorySegment stackArgsSegment = MemorySegment.allocateNative(stackArgsSize, 16);
420 MemoryAddress maOverflowArgArea = stackArgsSegment.baseAddress();
421 for (SimpleVaArg arg : stackArgs) {
422 if (arg.layout.byteSize() > 8) {
423 maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize()));
424 }
425 VarHandle writer = arg.varHandle();
426 writer.set(maOverflowArgArea, arg.value);
427 maOverflowArgArea = maOverflowArgArea.addOffset(arg.layout.byteSize());
428 }
429 stackArgsPtr = stackArgsSegment.baseAddress();
430 res.slices.add(stackArgsSegment);
431 }
432
433 MemoryAddress vaListAddr = vaListSegment.baseAddress();
434 VH_fp_offset.set(vaListAddr, (int) FP_OFFSET);
435 VH_overflow_arg_area.set(vaListAddr, stackArgsPtr);
436 VH_reg_save_area.set(vaListAddr, reg_save_area.baseAddress());
437 res.slices.add(reg_save_area);
438 assert reg_save_area.ownerThread() == vaListSegment.ownerThread();
439 return res;
440 }
441 }
442 }