1 /*
  2  *  Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
  3  *  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  *  This code is free software; you can redistribute it and/or modify it
  6  *  under the terms of the GNU General Public License version 2 only, as
  7  *  published by the Free Software Foundation.  Oracle designates this
  8  *  particular file as subject to the "Classpath" exception as provided
  9  *  by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  *  This code is distributed in the hope that it will be useful, but WITHOUT
 12  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  *  version 2 for more details (a copy is included in the LICENSE file that
 15  *  accompanied this code).
 16  *
 17  *  You should have received a copy of the GNU General Public License version
 18  *  2 along with this work; if not, write to the Free Software Foundation,
 19  *  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  *   Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  *  or visit www.oracle.com if you need additional information or have any
 23  *  questions.
 24  *
 25  */
 26 package jdk.internal.foreign.abi.x64.sysv;
 27 
 28 import jdk.incubator.foreign.CSupport;
 29 import jdk.incubator.foreign.GroupLayout;
 30 import jdk.incubator.foreign.MemoryAddress;
 31 import jdk.incubator.foreign.MemoryHandles;
 32 import jdk.incubator.foreign.MemoryLayout;
 33 import jdk.incubator.foreign.MemorySegment;
 34 import jdk.internal.foreign.Utils;
 35 import jdk.internal.foreign.abi.SharedUtils;
 36 
 37 import java.lang.invoke.VarHandle;
 38 import java.nio.ByteOrder;
 39 import java.util.ArrayList;
 40 import java.util.List;
 41 
 42 import static jdk.incubator.foreign.CSupport.SysV;
 43 import static jdk.incubator.foreign.CSupport.VaList;
 44 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 45 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 46 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
 47 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
 48 
 49 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists"
 50 public class SysVVaList implements VaList {
 51     static final Class<?> CARRIER = MemoryAddress.class;
 52 
 53 //    struct typedef __va_list_tag __va_list_tag {
 54 //        unsigned int               gp_offset;            /*     0     4 */
 55 //        unsigned int               fp_offset;            /*     4     4 */
 56 //        void *                     overflow_arg_area;    /*     8     8 */
 57 //        void *                     reg_save_area;        /*    16     8 */
 58 //
 59 //        /* size: 24, cachelines: 1, members: 4 */
 60 //        /* last cacheline: 24 bytes */
 61 //    };
 62     static final GroupLayout LAYOUT = MemoryLayout.ofStruct(
 63         SysV.C_INT.withName("gp_offset"),
 64         SysV.C_INT.withName("fp_offset"),
 65         SysV.C_POINTER.withName("overflow_arg_area"),
 66         SysV.C_POINTER.withName("reg_save_area")
 67     ).withName("__va_list_tag");
 68 
 69     private static final MemoryLayout GP_REG = MemoryLayout.ofValueBits(64, ByteOrder.nativeOrder());
 70     private static final MemoryLayout FP_REG = MemoryLayout.ofValueBits(128, ByteOrder.nativeOrder());
 71 
 72     private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.ofStruct(
 73         GP_REG.withName("%rdi"),
 74         GP_REG.withName("%rsi"),
 75         GP_REG.withName("%rdx"),
 76         GP_REG.withName("%rcx"),
 77         GP_REG.withName("%r8"),
 78         GP_REG.withName("%r9"),
 79         FP_REG.withName("%xmm0"),
 80         FP_REG.withName("%xmm1"),
 81         FP_REG.withName("%xmm2"),
 82         FP_REG.withName("%xmm3"),
 83         FP_REG.withName("%xmm4"),
 84         FP_REG.withName("%xmm5"),
 85         FP_REG.withName("%xmm6"),
 86         FP_REG.withName("%xmm7")
 87 // specification and implementation differ as to whether the following are part of a reg save area
 88 // Let's go with the implementation, since then it actually works :)
 89 //        FP_REG.withName("%xmm8"),
 90 //        FP_REG.withName("%xmm9"),
 91 //        FP_REG.withName("%xmm10"),
 92 //        FP_REG.withName("%xmm11"),
 93 //        FP_REG.withName("%xmm12"),
 94 //        FP_REG.withName("%xmm13"),
 95 //        FP_REG.withName("%xmm14"),
 96 //        FP_REG.withName("%xmm15")
 97     );
 98 
 99     private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0"));
100 
101     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
102     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
103 
104     private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used
105     private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs
106 
107     private static final VarHandle VH_fp_offset = LAYOUT.varHandle(int.class, groupElement("fp_offset"));
108     private static final VarHandle VH_gp_offset = LAYOUT.varHandle(int.class, groupElement("gp_offset"));
109     private static final VarHandle VH_overflow_arg_area
110         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("overflow_arg_area")));
111     private static final VarHandle VH_reg_save_area
112         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("reg_save_area")));
113 
114     private final MemorySegment segment;
115     private final List<MemorySegment> slices = new ArrayList<>();
116     private final MemorySegment regSaveArea;
117 
118     SysVVaList(MemorySegment segment) {
119         this.segment = segment;
120         regSaveArea = regSaveArea();
121         slices.add(regSaveArea);
122     }
123 
124     private int currentGPOffset() {
125         return (int) VH_gp_offset.get(segment.baseAddress());
126     }
127 
128     private void currentGPOffset(int i) {
129         VH_gp_offset.set(segment.baseAddress(), i);
130     }
131 
132     private int currentFPOffset() {
133         return (int) VH_fp_offset.get(segment.baseAddress());
134     }
135 
136     private void currentFPOffset(int i) {
137         VH_fp_offset.set(segment.baseAddress(), i);
138     }
139 
140     private MemoryAddress stackPtr() {
141         return (MemoryAddress) VH_overflow_arg_area.get(segment.baseAddress());
142     }
143 
144     private void stackPtr(MemoryAddress ptr) {
145         VH_overflow_arg_area.set(segment.baseAddress(), ptr);
146     }
147 
148     private MemorySegment regSaveArea() {
149         return MemorySegment.ofNativeRestricted((MemoryAddress) VH_reg_save_area.get(segment.baseAddress()),
150             LAYOUT_REG_SAVE_AREA.byteSize(), segment.ownerThread(), null, null);
151     }
152 
153     private void preAlignStack(MemoryLayout layout) {
154         if (layout.byteAlignment() > 8) {
155             stackPtr(Utils.alignUp(stackPtr(), 16));
156         }
157     }
158 
159     private void postAlignStack(MemoryLayout layout) {
160         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
161     }
162 
163     @Override
164     public int vargAsInt(MemoryLayout layout) {
165         return (int) read(int.class, layout);
166     }
167 
168     @Override
169     public long vargAsLong(MemoryLayout layout) {
170         return (long) read(long.class, layout);
171     }
172 
173     @Override
174     public double vargAsDouble(MemoryLayout layout) {
175         return (double) read(double.class, layout);
176     }
177 
178     @Override
179     public MemoryAddress vargAsAddress(MemoryLayout layout) {
180         return (MemoryAddress) read(MemoryAddress.class, layout);
181     }
182 
183     @Override
184     public MemorySegment vargAsSegment(MemoryLayout layout) {
185         return (MemorySegment) read(MemorySegment.class, layout);
186     }
187 
188     private Object read(Class<?> carrier, MemoryLayout layout) {
189         checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
190         TypeClass typeClass = TypeClass.classifyLayout(layout);
191         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
192             preAlignStack(layout);
193             return switch (typeClass.kind()) {
194                 case STRUCT -> {
195                     try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
196                                                                                 segment.ownerThread(), null, null)) {
197                         MemorySegment seg = MemorySegment.allocateNative(layout);
198                         seg.copyFrom(slice);
199                         postAlignStack(layout);
200                         yield seg;
201                     }
202                 }
203                 case POINTER, INTEGER, FLOAT -> {
204                     VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
205                     try (MemorySegment slice = MemorySegment.ofNativeRestricted(stackPtr(), layout.byteSize(),
206                                                                                 segment.ownerThread(), null, null)) {
207                         Object res = reader.get(slice.baseAddress());
208                         postAlignStack(layout);
209                         yield res;
210                     }
211                 }
212             };
213         } else {
214             return switch (typeClass.kind()) {
215                 case STRUCT -> {
216                     MemorySegment value = MemorySegment.allocateNative(layout);
217                     int classIdx = 0;
218                     long offset = 0;
219                     while (offset < layout.byteSize()) {
220                         final long copy = Math.min(layout.byteSize() - offset, 8);
221                         boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
222                         MemorySegment slice = value.asSlice(offset, copy);
223                         if (isSSE) {
224                             slice.copyFrom(regSaveArea.asSlice(currentFPOffset(), copy));
225                             currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
226                         } else {
227                             slice.copyFrom(regSaveArea.asSlice(currentGPOffset(), copy));
228                             currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
229                         }
230                         offset += copy;
231                     }
232                     yield value;
233                 }
234                 case POINTER, INTEGER -> {
235                     VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
236                     Object res = reader.get(regSaveArea.baseAddress().addOffset(currentGPOffset()));
237                     currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
238                     yield res;
239                 }
240                 case FLOAT -> {
241                     VarHandle reader = layout.varHandle(carrier);
242                     Object res = reader.get(regSaveArea.baseAddress().addOffset(currentFPOffset()));
243                     currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
244                     yield res;
245                 }
246             };
247         }
248     }
249 
250     @Override
251     public void skip(MemoryLayout... layouts) {
252         for (MemoryLayout layout : layouts) {
253             TypeClass typeClass = TypeClass.classifyLayout(layout);
254             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
255                 preAlignStack(layout);
256                 postAlignStack(layout);
257             } else {
258                 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE));
259                 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE));
260             }
261         }
262     }
263 
264     static SysVVaList.Builder builder() {
265         return new SysVVaList.Builder();
266     }
267 
268     public static VaList ofAddress(MemoryAddress ma) {
269         return new SysVVaList(MemorySegment.ofNativeRestricted(ma, LAYOUT.byteSize(), Thread.currentThread(), null, null));
270     }
271 
272     MemorySegment getSegment() {
273         return segment;
274     }
275 
276     @Override
277     public boolean isAlive() {
278         return segment.isAlive();
279     }
280 
281     @Override
282     public void close() {
283         segment.close();
284         slices.forEach(MemorySegment::close);
285     }
286 
287     @Override
288     public VaList copy() {
289         MemorySegment copy = MemorySegment.allocateNative(LAYOUT.byteSize());
290         copy.copyFrom(segment);
291         return new SysVVaList(copy);
292     }
293 
294     @Override
295     public MemoryAddress address() {
296         return segment.baseAddress();
297     }
298 
299     private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) {
300         return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE
301                 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE;
302     }
303 
304     @Override
305     public String toString() {
306         return "SysVVaList{"
307                + "gp_offset=" + currentGPOffset()
308                + ", fp_offset=" + currentFPOffset()
309                + ", overflow_arg_area=" + stackPtr()
310                + ", reg_save_area=" + regSaveArea()
311                + '}';
312     }
313 
314     static class Builder implements CSupport.VaList.Builder {
315         private final MemorySegment reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA);
316         private long currentGPOffset = 0;
317         private long currentFPOffset = FP_OFFSET;
318         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
319 
320         @Override
321         public Builder vargFromInt(MemoryLayout layout, int value) {
322             return arg(int.class, layout, value);
323         }
324 
325         @Override
326         public Builder vargFromLong(MemoryLayout layout, long value) {
327             return arg(long.class, layout, value);
328         }
329 
330         @Override
331         public Builder vargFromDouble(MemoryLayout layout, double value) {
332             return arg(double.class, layout, value);
333         }
334 
335         @Override
336         public Builder vargFromAddress(MemoryLayout layout, MemoryAddress value) {
337             return arg(MemoryAddress.class, layout, value);
338         }
339 
340         @Override
341         public Builder vargFromSegment(MemoryLayout layout, MemorySegment value) {
342             return arg(MemorySegment.class, layout, value);
343         }
344 
345         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
346             checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
347             TypeClass typeClass = TypeClass.classifyLayout(layout);
348             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)) {
349                 // stack it!
350                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
351             } else {
352                 switch (typeClass.kind()) {
353                     case STRUCT -> {
354                         MemorySegment valueSegment = (MemorySegment) value;
355                         int classIdx = 0;
356                         long offset = 0;
357                         while (offset < layout.byteSize()) {
358                             final long copy = Math.min(layout.byteSize() - offset, 8);
359                             boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
360                             MemorySegment slice = valueSegment.asSlice(offset, copy);
361                             if (isSSE) {
362                                 reg_save_area.asSlice(currentFPOffset, copy).copyFrom(slice);
363                                 currentFPOffset += FP_SLOT_SIZE;
364                             } else {
365                                 reg_save_area.asSlice(currentGPOffset, copy).copyFrom(slice);
366                                 currentGPOffset += GP_SLOT_SIZE;
367                             }
368                             offset += copy;
369                         }
370                     }
371                     case POINTER, INTEGER -> {
372                         VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
373                         writer.set(reg_save_area.baseAddress().addOffset(currentGPOffset), value);
374                         currentGPOffset += GP_SLOT_SIZE;
375                     }
376                     case FLOAT -> {
377                         VarHandle writer = layout.varHandle(carrier);
378                         writer.set(reg_save_area.baseAddress().addOffset(currentFPOffset), value);
379                         currentFPOffset += FP_SLOT_SIZE;
380                     }
381                 }
382             }
383             return this;
384         }
385 
386         public SysVVaList build() {
387             MemorySegment vaListSegment = MemorySegment.allocateNative(LAYOUT.byteSize());
388             SysVVaList res = new SysVVaList(vaListSegment);
389             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
390             if (!stackArgs.isEmpty()) {
391                 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum);
392                 MemorySegment stackArgsSegment = MemorySegment.allocateNative(stackArgsSize, 16);
393                 MemoryAddress maOverflowArgArea = stackArgsSegment.baseAddress();
394                 for (SimpleVaArg arg : stackArgs) {
395                     if (arg.layout.byteSize() > 8) {
396                         maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize()));
397                     }
398                     VarHandle writer = arg.varHandle();
399                     writer.set(maOverflowArgArea, arg.value);
400                     maOverflowArgArea = maOverflowArgArea.addOffset(arg.layout.byteSize());
401                 }
402                 stackArgsPtr = stackArgsSegment.baseAddress();
403                 res.slices.add(stackArgsSegment);
404             }
405 
406             MemoryAddress vaListAddr = vaListSegment.baseAddress();
407             VH_fp_offset.set(vaListAddr, (int) FP_OFFSET);
408             VH_overflow_arg_area.set(vaListAddr, stackArgsPtr);
409             VH_reg_save_area.set(vaListAddr, reg_save_area.baseAddress());
410             res.slices.add(reg_save_area);
411             assert reg_save_area.ownerThread() == vaListSegment.ownerThread();
412             return res;
413         }
414     }
415 }