OpenJDK / jdk / jdk
changeset 57115:88502b1cf76f
8230765: Implement nmethod barrier for x86_32 platforms
Reviewed-by: rkennke, eosterlund
author | zgu |
---|---|
date | Mon, 09 Sep 2019 11:43:16 -0400 |
parents | 78aa7484c722 |
children | 7799a51dbe30 |
files | src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp src/hotspot/cpu/x86/stubGenerator_x86_32.cpp src/hotspot/cpu/x86/stubRoutines_x86.hpp src/hotspot/cpu/x86/stubRoutines_x86_32.cpp src/hotspot/share/gc/shared/barrierSetNMethod.cpp src/hotspot/share/gc/shared/barrierSetNMethod.hpp src/hotspot/share/gc/z/zBarrierSetNMethod.cpp src/hotspot/share/gc/z/zBarrierSetNMethod.hpp |
diffstat | 10 files changed, 152 insertions(+), 32 deletions(-) [+] |
line wrap: on
line diff
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -327,24 +327,42 @@ #endif } +#ifdef _LP64 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); if (bs_nm == NULL) { return; } -#ifndef _LP64 - ShouldNotReachHere(); -#else Label continuation; - Register thread = LP64_ONLY(r15_thread); + Register thread = r15_thread; Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset())); __ align(8); __ cmpl(disarmed_addr, 0); __ jcc(Assembler::equal, continuation); __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); __ bind(continuation); +} +#else +void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm == NULL) { + return; + } + + Label continuation; + + Register tmp = rdi; + __ push(tmp); + __ movptr(tmp, (intptr_t)bs_nm->disarmed_value_address()); + Address disarmed_addr(tmp, 0); + __ align(4); + __ cmpl(disarmed_addr, 0); + __ pop(tmp); + __ jcc(Assembler::equal, continuation); + __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); + __ bind(continuation); +} #endif -} void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,6 +35,7 @@ class NativeNMethodCmpBarrier: public NativeInstruction { public: +#ifdef _LP64 enum Intel_specific_constants { instruction_code = 0x81, instruction_size = 8, @@ -42,6 +43,14 @@ instruction_rex_prefix = Assembler::REX | Assembler::REX_B, instruction_modrm = 0x7f // [r15 + offset] }; +#else + enum Intel_specific_constants { + instruction_code = 0x81, + instruction_size = 7, + imm_offset = 2, + instruction_modrm = 0x3f // [rdi] + }; +#endif address instruction_address() const { return addr_at(0); } address immediate_address() const { return addr_at(imm_offset); } @@ -51,6 +60,7 @@ void verify() const; }; +#ifdef _LP64 void NativeNMethodCmpBarrier::verify() const { if (((uintptr_t) instruction_address()) & 0x7) { fatal("Not properly aligned"); @@ -77,6 +87,27 @@ fatal("not a cmp barrier"); } } +#else +void NativeNMethodCmpBarrier::verify() const { + if (((uintptr_t) instruction_address()) & 0x3) { + fatal("Not properly aligned"); + } + + int inst = ubyte_at(0); + if (inst != instruction_code) { + tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", p2i(instruction_address()), + inst); + fatal("not a cmp barrier"); + } + + int modrm = ubyte_at(1); + if (modrm != instruction_modrm) { + tty->print_cr("Addr: " INTPTR_FORMAT " mod/rm: 0x%x", p2i(instruction_address()), + modrm); + fatal("not a cmp barrier"); + } +} +#endif // _LP64 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) { /* @@ -127,7 +158,7 @@ // NativeNMethodCmpBarrier::verify() will immediately complain when it does // not find the expected native instruction at this offset, which needs updating. // Note that this offset is invariant of PreserveFramePointer. -static const int entry_barrier_offset = -19; +static const int entry_barrier_offset = LP64_ONLY(-19) NOT_LP64(-18); static NativeNMethodCmpBarrier* native_nmethod_barrier(nmethod* nm) { address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset;
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -975,6 +975,9 @@ address c2i_entry = __ pc(); + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->c2i_entry_barrier(masm); + gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); __ flush(); @@ -1886,6 +1889,10 @@ // -2 because return address is already present and so is saved rbp __ subptr(rsp, stack_size - 2*wordSize); + + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->nmethod_entry_barrier(masm); + // Frame is now completed as far as size and linkage. int frame_complete = ((intptr_t)__ pc()) - start; @@ -1921,12 +1928,12 @@ // if we load it once it is usable thru the entire wrapper const Register thread = rdi; - // We use rsi as the oop handle for the receiver/klass - // It is callee save so it survives the call to native - - const Register oop_handle_reg = rsi; - - __ get_thread(thread); + // We use rsi as the oop handle for the receiver/klass + // It is callee save so it survives the call to native + + const Register oop_handle_reg = rsi; + + __ get_thread(thread); if (is_critical_native && !Universe::heap()->supports_object_pinning()) { check_needs_gc_for_critical_native(masm, thread, stack_slots, total_c_args, total_in_args,
--- a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -27,6 +27,7 @@ #include "asm/macroAssembler.inline.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "interpreter/interpreter.hpp" #include "memory/universe.hpp" #include "nativeInst_x86.hpp" @@ -3663,6 +3664,68 @@ __ ret(0); } + address generate_method_entry_barrier() { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); + + Label deoptimize_label; + + address start = __ pc(); + + __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing + + BLOCK_COMMENT("Entry:"); + __ enter(); // save rbp + + // save rbx, because we want to use that value. + // We could do without it but then we depend on the number of slots used by pusha + __ push(rbx); + + __ lea(rbx, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for rbx - this should be the return address + + __ pusha(); + + // xmm0 and xmm1 may be used for passing float/double arguments + const int xmm_size = wordSize * 2; + const int xmm_spill_size = xmm_size * 2; + __ subptr(rsp, xmm_spill_size); + __ movdqu(Address(rsp, xmm_size * 1), xmm1); + __ movdqu(Address(rsp, xmm_size * 0), xmm0); + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), rbx); + + __ movdqu(xmm0, Address(rsp, xmm_size * 0)); + __ movdqu(xmm1, Address(rsp, xmm_size * 1)); + __ addptr(rsp, xmm_spill_size); + + __ cmpl(rax, 1); // 1 means deoptimize + __ jcc(Assembler::equal, deoptimize_label); + + __ popa(); + __ pop(rbx); + + __ leave(); + + __ addptr(rsp, 1 * wordSize); // cookie + __ ret(0); + + __ BIND(deoptimize_label); + + __ popa(); + __ pop(rbx); + + __ leave(); + + // this can be taken out, but is good for verification purposes. getting a SIGSEGV + // here while still having a correct stack is valuable + __ testptr(rsp, Address(rsp, 0)); + + __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier + __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point + + return start; + } + public: // Information about frame layout at time of blocking runtime call. // Note that we only have to preserve callee-saved registers since @@ -3959,6 +4022,11 @@ StubRoutines::_safefetchN_entry = StubRoutines::_safefetch32_entry; StubRoutines::_safefetchN_fault_pc = StubRoutines::_safefetch32_fault_pc; StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc; + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier(); + } }
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp Mon Sep 09 11:43:16 2019 -0400 @@ -55,14 +55,8 @@ static address _double_sign_mask; static address _double_sign_flip; - static address _method_entry_barrier; - public: - static address method_entry_barrier() { - return _method_entry_barrier; - } - static address get_previous_fp_entry() { return _get_previous_fp_entry; } @@ -121,6 +115,8 @@ //shuffle mask for big-endian 128-bit integers static address _counter_shuffle_mask_addr; + static address _method_entry_barrier; + // masks and table for CRC32 static uint64_t _crc_by128_masks[]; static juint _crc_table[]; @@ -221,6 +217,7 @@ static address upper_word_mask_addr() { return _upper_word_mask_addr; } static address shuffle_byte_flip_mask_addr() { return _shuffle_byte_flip_mask_addr; } static address k256_addr() { return _k256_adr; } + static address method_entry_barrier() { return _method_entry_barrier; } static address vector_short_to_byte_mask() { return _vector_short_to_byte_mask;
--- a/src/hotspot/cpu/x86/stubRoutines_x86_32.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/cpu/x86/stubRoutines_x86_32.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,3 +32,5 @@ // a description of how to extend it, see the stubRoutines.hpp file. address StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = NULL; +address StubRoutines::x86::_method_entry_barrier = NULL; +
--- a/src/hotspot/share/gc/shared/barrierSetNMethod.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/share/gc/shared/barrierSetNMethod.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -32,9 +32,7 @@ #include "utilities/debug.hpp" int BarrierSetNMethod::disarmed_value() const { - char* disarmed_addr = reinterpret_cast<char*>(Thread::current()); - disarmed_addr += in_bytes(thread_disarmed_offset()); - return *reinterpret_cast<int*>(disarmed_addr); + return *disarmed_value_address(); } bool BarrierSetNMethod::supports_entry_barrier(nmethod* nm) {
--- a/src/hotspot/share/gc/shared/barrierSetNMethod.hpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/share/gc/shared/barrierSetNMethod.hpp Mon Sep 09 11:43:16 2019 -0400 @@ -34,13 +34,14 @@ class BarrierSetNMethod: public CHeapObj<mtGC> { bool supports_entry_barrier(nmethod* nm); void deoptimize(nmethod* nm, address* return_addr_ptr); + int disarmed_value() const; protected: - virtual int disarmed_value() const; virtual bool nmethod_entry_barrier(nmethod* nm) = 0; public: virtual ByteSize thread_disarmed_offset() const = 0; + virtual int* disarmed_value_address() const = 0; static int nmethod_stub_entry_barrier(address* return_address_ptr); bool nmethod_osr_entry_barrier(nmethod* nm);
--- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp Mon Sep 09 11:43:16 2019 -0400 @@ -61,12 +61,10 @@ return true; } -int ZBarrierSetNMethod::disarmed_value() const { - // We override the default BarrierSetNMethod::disarmed_value() since - // this can be called by GC threads, which doesn't keep an up to date - // address_bad_mask. - const uintptr_t disarmed_addr = ((uintptr_t)&ZAddressBadMask) + ZNMethodDisarmedOffset; - return *((int*)disarmed_addr); +int* ZBarrierSetNMethod::disarmed_value_address() const { + const uintptr_t mask_addr = reinterpret_cast<uintptr_t>(&ZAddressBadMask); + const uintptr_t disarmed_addr = mask_addr + ZNMethodDisarmedOffset; + return reinterpret_cast<int*>(disarmed_addr); } ByteSize ZBarrierSetNMethod::thread_disarmed_offset() const {
--- a/src/hotspot/share/gc/z/zBarrierSetNMethod.hpp Tue Nov 26 14:33:56 2019 -0500 +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.hpp Mon Sep 09 11:43:16 2019 -0400 @@ -31,11 +31,11 @@ class ZBarrierSetNMethod : public BarrierSetNMethod { protected: - virtual int disarmed_value() const; virtual bool nmethod_entry_barrier(nmethod* nm); public: virtual ByteSize thread_disarmed_offset() const; + virtual int* disarmed_value_address() const; }; #endif // SHARE_GC_Z_ZBARRIERSETNMETHOD_HPP