OpenJDK / portola / portola
changeset 8364:07cd2e4eb172
Merge
| author | jgodinez |
|---|---|
| date | Fri, 18 Feb 2011 12:04:36 -0800 |
| parents | 63664c9ccde0 3edb234b012f |
| children | 803a1fd60835 |
| files | hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp jdk/make/java/hpi/Makefile jdk/make/java/hpi/hpi_common.gmk jdk/make/java/hpi/native/Makefile jdk/make/java/hpi/native/mapfile-vers jdk/make/java/hpi/native/reorder-i586 jdk/make/java/hpi/native/reorder-sparc jdk/make/java/hpi/native/reorder-sparcv9 jdk/make/java/hpi/windows/Makefile jdk/src/share/hpi/export/bool.h jdk/src/share/hpi/export/dll.h jdk/src/share/hpi/export/hpi.h jdk/src/share/hpi/include/hpi_impl.h jdk/src/share/hpi/include/vm_calls.h jdk/src/share/hpi/src/hpi.c jdk/src/solaris/hpi/export/byteorder_md.h jdk/src/solaris/hpi/export/hpi_md.h jdk/src/solaris/hpi/export/io_md.h jdk/src/solaris/hpi/export/path_md.h jdk/src/solaris/hpi/export/timeval_md.h jdk/src/solaris/hpi/include/hpi_init.h jdk/src/solaris/hpi/include/interrupt.h jdk/src/solaris/hpi/include/largefile.h jdk/src/solaris/hpi/include/largefile_linux.h jdk/src/solaris/hpi/include/largefile_solaris.h jdk/src/solaris/hpi/native_threads/include/condvar_md.h jdk/src/solaris/hpi/native_threads/include/monitor_md.h jdk/src/solaris/hpi/native_threads/include/mutex_md.h jdk/src/solaris/hpi/native_threads/include/np.h jdk/src/solaris/hpi/native_threads/include/porting.h jdk/src/solaris/hpi/native_threads/include/threads_md.h jdk/src/solaris/hpi/native_threads/src/condvar_md.c jdk/src/solaris/hpi/native_threads/src/interrupt_md.c jdk/src/solaris/hpi/native_threads/src/monitor_md.c jdk/src/solaris/hpi/native_threads/src/mutex_md.c jdk/src/solaris/hpi/native_threads/src/sys_api_td.c jdk/src/solaris/hpi/native_threads/src/threads_linux.c jdk/src/solaris/hpi/native_threads/src/threads_md.c jdk/src/solaris/hpi/native_threads/src/threads_solaris.c jdk/src/solaris/hpi/src/interrupt.c jdk/src/solaris/hpi/src/linker_md.c jdk/src/solaris/hpi/src/memory_md.c jdk/src/solaris/hpi/src/system_md.c jdk/src/windows/hpi/export/byteorder_md.h jdk/src/windows/hpi/export/hpi_md.h jdk/src/windows/hpi/export/io_md.h jdk/src/windows/hpi/export/path_md.h jdk/src/windows/hpi/export/timeval_md.h jdk/src/windows/hpi/include/monitor_md.h jdk/src/windows/hpi/include/mutex_md.h jdk/src/windows/hpi/include/threads_md.h jdk/src/windows/hpi/src/linker_md.c jdk/src/windows/hpi/src/memory_md.c jdk/src/windows/hpi/src/monitor_md.c jdk/src/windows/hpi/src/path_md.c jdk/src/windows/hpi/src/socket_md.c jdk/src/windows/hpi/src/sys_api_md.c jdk/src/windows/hpi/src/system_md.c jdk/src/windows/hpi/src/threads_md.c jdk/test/java/net/InetAddress/B4762344.java jdk/test/java/net/InetAddress/META-INF/services/sun.net.spi.nameservice.NameServiceDescriptor jdk/test/java/net/InetAddress/Simple1NameServiceDescriptor.java jdk/test/java/net/InetAddress/Simple2NameServiceDescriptor.java jdk/test/java/net/InetAddress/SimpleNameService.java jdk/test/sun/net/InetAddress/nameservice/B6442088.java jdk/test/sun/net/InetAddress/nameservice/CacheTest.java jdk/test/sun/net/InetAddress/nameservice/META-INF/services/sun.net.spi.nameservice.NameServiceDescriptor jdk/test/sun/net/InetAddress/nameservice/SimpleNameService.java jdk/test/sun/net/InetAddress/nameservice/SimpleNameServiceDescriptor.java langtools/src/share/classes/com/sun/tools/classfile/ExtendedAnnotation.java langtools/src/share/classes/com/sun/tools/classfile/RuntimeInvisibleTypeAnnotations_attribute.java langtools/src/share/classes/com/sun/tools/classfile/RuntimeTypeAnnotations_attribute.java langtools/src/share/classes/com/sun/tools/classfile/RuntimeVisibleTypeAnnotations_attribute.java langtools/src/share/classes/com/sun/tools/javac/code/TypeAnnotations.java langtools/src/share/classes/com/sun/tools/javap/TypeAnnotationWriter.java langtools/test/tools/javac/diags/examples/EnumConstRequired.java langtools/test/tools/javac/diags/examples/TypeParameterOnPolymorphicSignature.java langtools/test/tools/javac/meth/InvokeDynTrans.out langtools/test/tools/javac/meth/InvokeMHTrans.java langtools/test/tools/javac/meth/InvokeMHTrans.out |
| diffstat | 633 files changed, 16272 insertions(+), 22806 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgtags Fri Feb 18 11:15:40 2011 -0800 +++ b/.hgtags Fri Feb 18 12:04:36 2011 -0800 @@ -100,3 +100,5 @@ 142129d8599d1f56b29387e7f9a5fad53b6d61df jdk7-b123 aa894c225b1a517b665ac2a58295217ea2245134 jdk7-b124 f658ec2730fa29323c36d23c27e54c7219ef5e16 jdk7-b125 +f1df068076986679ea1105532a65529d63a89060 jdk7-b126 +f83cd8bd35c678f94e526990e03dc838d0ec2717 jdk7-b127
--- a/.hgtags-top-repo Fri Feb 18 11:15:40 2011 -0800 +++ b/.hgtags-top-repo Fri Feb 18 12:04:36 2011 -0800 @@ -100,3 +100,5 @@ ed6950da30cf1e8904b4bdb034d471647942271f jdk7-b123 024a6755895bf91b5a3c98984c89ee018efbf538 jdk7-b124 5c4df7e992775c102f08e9f1c0a124b324641b70 jdk7-b125 +b566d490905691787f8931f69947a92c67c6d5e4 jdk7-b126 +bd70f76b0309068f157ae759c36eac8f2c6d098e jdk7-b127
--- a/Makefile Fri Feb 18 11:15:40 2011 -0800 +++ b/Makefile Fri Feb 18 12:04:36 2011 -0800 @@ -29,15 +29,6 @@ TOPDIR:=. endif -# Openjdk sources (only used if SKIP_OPENJDK_BUILD!=true) -OPENJDK_SOURCETREE=$(TOPDIR)/openjdk -OPENJDK_BUILDDIR:=$(shell \ - if [ -r $(OPENJDK_SOURCETREE)/Makefile ]; then \ - echo "$(OPENJDK_SOURCETREE)"; \ - else \ - echo "."; \ - fi) - ifndef JDK_TOPDIR JDK_TOPDIR=$(TOPDIR)/jdk endif @@ -70,7 +61,7 @@ all:: @$(START_ECHO) -all:: openjdk_check sanity +all:: sanity ifeq ($(SKIP_FASTDEBUG_BUILD), false) all:: fastdebug_build @@ -80,10 +71,6 @@ all:: debug_build endif -ifneq ($(SKIP_OPENJDK_BUILD), true) - all:: openjdk_build -endif - all:: all_product_build all:: @@ -267,81 +254,6 @@ debug_build:: build_debug_image fastdebug_build:: build_fastdebug_image -# Check on whether we really can build the openjdk, need source etc. -openjdk_check: FRC -ifneq ($(SKIP_OPENJDK_BUILD), true) - @$(ECHO) " " - @$(ECHO) "=================================================" - @if [ ! -r $(OPENJDK_BUILDDIR)/Makefile ] ; then \ - $(ECHO) "ERROR: No openjdk source tree available at: $(OPENJDK_BUILDDIR)"; \ - exit 1; \ - else \ - $(ECHO) "OpenJDK will be built after JDK is built"; \ - $(ECHO) " OPENJDK_BUILDDIR=$(OPENJDK_BUILDDIR)"; \ - fi - @$(ECHO) "=================================================" - @$(ECHO) " " -endif - -# If we have bundle rules, we have a chance here to do a complete cycle -# build, of production and open build. -# FIXUP: We should create the openjdk source bundle and build that? -# But how do we reliable create or get at a formal openjdk source tree? -# The one we have needs to be trimmed of built bits and closed dirs. -# The repositories might not be available. -# The openjdk source bundle is probably not available. - -ifneq ($(SKIP_OPENJDK_BUILD), true) - ifeq ($(BUILD_JDK), true) - ifeq ($(BUNDLE_RULES_AVAILABLE), true) - -OPENJDK_OUTPUTDIR=$(ABS_OUTPUTDIR)/open-output -OPENJDK_BUILD_NAME \ - = openjdk-$(JDK_MINOR_VERSION)-$(BUILD_NUMBER)-$(PLATFORM)-$(ARCH)-$(BUNDLE_DATE) -OPENJDK_BUILD_BINARY_ZIP=$(ABS_BIN_BUNDLEDIR)/$(OPENJDK_BUILD_NAME).zip -BUILT_IMAGE=$(ABS_OUTPUTDIR)/j2sdk-image -ifeq ($(PLATFORM)$(ARCH_DATA_MODEL),solaris64) - OPENJDK_BOOTDIR=$(BOOTDIR) - OPENJDK_IMPORTJDK=$(JDK_IMPORT_PATH) -else - OPENJDK_BOOTDIR=$(BUILT_IMAGE) - OPENJDK_IMPORTJDK=$(BUILT_IMAGE) -endif - -openjdk_build: - @$(START_ECHO) - @$(ECHO) " " - @$(ECHO) "=================================================" - @$(ECHO) "Starting openjdk build" - @$(ECHO) " Using: ALT_JDK_DEVTOOLS_DIR=$(JDK_DEVTOOLS_DIR)" - @$(ECHO) "=================================================" - @$(ECHO) " " - $(RM) -r $(OPENJDK_OUTPUTDIR) - $(MKDIR) -p $(OPENJDK_OUTPUTDIR) - ($(CD) $(OPENJDK_BUILDDIR) && $(MAKE) \ - OPENJDK=true \ - GENERATE_DOCS=false \ - ALT_JDK_DEVTOOLS_DIR=$(JDK_DEVTOOLS_DIR) \ - ALT_OUTPUTDIR=$(OPENJDK_OUTPUTDIR) \ - ALT_BOOTDIR=$(OPENJDK_BOOTDIR) \ - ALT_JDK_IMPORT_PATH=$(OPENJDK_IMPORTJDK) \ - product_build ) - $(RM) $(OPENJDK_BUILD_BINARY_ZIP) - ( $(CD) $(OPENJDK_OUTPUTDIR)/j2sdk-image && \ - $(ZIPEXE) -q -r $(OPENJDK_BUILD_BINARY_ZIP) .) - $(RM) -r $(OPENJDK_OUTPUTDIR) - @$(ECHO) " " - @$(ECHO) "=================================================" - @$(ECHO) "Finished openjdk build" - @$(ECHO) " Binary Bundle: $(OPENJDK_BUILD_BINARY_ZIP)" - @$(ECHO) "=================================================" - @$(ECHO) " " - @$(FINISH_ECHO) - - endif - endif -endif - clobber:: $(RM) -r $(OUTPUTDIR)/* $(RM) -r $(OUTPUTDIR)/../$(PLATFORM)-$(ARCH)-debug/* @@ -448,7 +360,6 @@ DEVTOOLS_PATH.desc = Directory containing zip and gnumake CUPS_HEADERS_PATH.desc = Include directory location for CUPS header files DXSDK_PATH.desc = Root directory of DirectX SDK -MSVCRT_DLL_PATH.desc = Directory containing mscvrt.dll # Make variables to print out (description and value) VARIABLE_PRINTVAL_LIST += \ @@ -477,12 +388,10 @@ ifeq ($(PLATFORM), windows) VARIABLE_PRINTVAL_LIST += \ - DXSDK_PATH \ - MSVCRT_DLL_PATH + DXSDK_PATH VARIABLE_CHECKDIR_LIST += \ - DXSDK_PATH \ - MSVCRT_DLL_PATH + DXSDK_PATH endif
--- a/README-builds.html Fri Feb 18 11:15:40 2011 -0800 +++ b/README-builds.html Fri Feb 18 12:04:36 2011 -0800 @@ -1414,14 +1414,14 @@ but it's normally found via the DirectX environment variable <tt>DXSDK_DIR</tt>. </blockquote> - <strong><a name="msvcrt"><tt>MSVCR100.DLL</tt></a></strong> + <strong><a name="msvcrNN"><tt>MSVCR100.DLL</tt></a></strong> <blockquote> The OpenJDK build requires access to a redistributable <tt>MSVCR100.DLL</tt>. This is usually picked up automatically from the redist directories of Visual Studio 2010. If this cannot be found set the - <a href="#ALT_MSVCRT_DLL_PATH"><tt>ALT_MSVCRT_DLL_PATH</tt></a> + <a href="#ALT_MSVCRNN_DLL_PATH"><tt>ALT_MSVCRNN_DLL_PATH</tt></a> variable to the location of this file. <p> </blockquote> @@ -1671,15 +1671,10 @@ variable <tt>DXSDK_DIR</tt>, failing that, look in <tt>C:/DXSDK</tt>. </dd> - <dt><tt><a name="ALT_MSVCRT_DLL_PATH">ALT_MSVCRT_DLL_PATH</a></tt> </dt> - <dd> - The location of the - <a href="#msvcrt"><tt>MSVCRT.DLL</tt></a>. - </dd> <dt><tt><a name="ALT_MSVCRNN_DLL_PATH">ALT_MSVCRNN_DLL_PATH</a></tt> </dt> <dd> The location of the - <a href="#msvcrt"><tt>MSVCR100.DLL</tt></a>. + <a href="#msvcrNN"><tt>MSVCR100.DLL</tt></a>. </dd> </dl> </dd>
--- a/corba/.hgtags Fri Feb 18 11:15:40 2011 -0800 +++ b/corba/.hgtags Fri Feb 18 12:04:36 2011 -0800 @@ -100,3 +100,5 @@ a230c142628cea22475ab9dc5cd544266ddf2466 jdk7-b123 f90b3e014e831eb4f32ef035a1dad2b8ba87949f jdk7-b124 1ce58c72b7892cb813eb920276c7e7f17a1b79fe jdk7-b125 +d7532bcd3742f1576dd07ff9fbb535c9c9a276e9 jdk7-b126 +64775e83f4df894355f45555f50c410de6727b4e jdk7-b127
--- a/hotspot/.hgtags Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/.hgtags Fri Feb 18 12:04:36 2011 -0800 @@ -141,3 +141,6 @@ 0a8e0d4345b37b71ec49dda08ee03b68c4f1b592 jdk7-b124 0a8e0d4345b37b71ec49dda08ee03b68c4f1b592 hs20-b05 e24ab3fa6aafad3efabbe7dba9918c5f461a20b1 jdk7-b125 +4c851c931d001a882cab809aaf3a55371b919244 jdk7-b126 +e24ab3fa6aafad3efabbe7dba9918c5f461a20b1 hs20-b06 +102466e70debc4b907afbd7624e34ddb1aafee9f jdk7-b127
--- a/hotspot/make/hotspot_version Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/make/hotspot_version Fri Feb 18 12:04:36 2011 -0800 @@ -35,7 +35,7 @@ HS_MAJOR_VER=20 HS_MINOR_VER=0 -HS_BUILD_NUMBER=06 +HS_BUILD_NUMBER=07 JDK_MAJOR_VER=1 JDK_MINOR_VER=7
--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -4104,7 +4104,7 @@ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t1); sub(top, t1, t1); // size of tlab's allocated portion - incr_allocated_bytes(t1, 0, t2); + incr_allocated_bytes(t1, t2, t3); // refill the tlab with an eden allocation bind(do_refill); @@ -4138,19 +4138,14 @@ delayed()->nop(); } -void MacroAssembler::incr_allocated_bytes(Register var_size_in_bytes, - int con_size_in_bytes, - Register t1) { +void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, + Register t1, Register t2) { // Bump total bytes allocated by this thread assert(t1->is_global(), "must be global reg"); // so all 64 bits are saved on a context switch - assert_different_registers(var_size_in_bytes, t1); + assert_different_registers(size_in_bytes.register_or_noreg(), t1, t2); // v8 support has gone the way of the dodo ldx(G2_thread, in_bytes(JavaThread::allocated_bytes_offset()), t1); - if (var_size_in_bytes->is_valid()) { - add(t1, var_size_in_bytes, t1); - } else { - add(t1, con_size_in_bytes, t1); - } + add(t1, ensure_simm13_or_reg(size_in_bytes, t2), t1); stx(t1, G2_thread, in_bytes(JavaThread::allocated_bytes_offset())); }
--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -823,15 +823,23 @@ }; // test if x is within signed immediate range for nbits - static bool is_simm(int x, int nbits) { return -( 1 << nbits-1 ) <= x && x < ( 1 << nbits-1 ); } + static bool is_simm(intptr_t x, int nbits) { return -( intptr_t(1) << nbits-1 ) <= x && x < ( intptr_t(1) << nbits-1 ); } // test if -4096 <= x <= 4095 - static bool is_simm13(int x) { return is_simm(x, 13); } + static bool is_simm13(intptr_t x) { return is_simm(x, 13); } + + static bool is_in_wdisp_range(address a, address b, int nbits) { + intptr_t d = intptr_t(b) - intptr_t(a); + return is_simm(d, nbits + 2); + } // test if label is in simm16 range in words (wdisp16). bool is_in_wdisp16_range(Label& L) { - intptr_t d = intptr_t(pc()) - intptr_t(target(L)); - return is_simm(d, 18); + return is_in_wdisp_range(target(L), pc(), 16); + } + // test if the distance between two addresses fits in simm30 range in words + static bool is_in_wdisp30_range(address a, address b) { + return is_in_wdisp_range(a, b, 30); } enum ASIs { // page 72, v9 @@ -1843,6 +1851,8 @@ inline void jmp( Register s1, Register s2 ); inline void jmp( Register s1, int simm13a, RelocationHolder const& rspec = RelocationHolder() ); + // Check if the call target is out of wdisp30 range (relative to the code cache) + static inline bool is_far_target(address d); inline void call( address d, relocInfo::relocType rt = relocInfo::runtime_call_type ); inline void call( Label& L, relocInfo::relocType rt = relocInfo::runtime_call_type ); inline void callr( Register s1, Register s2 ); @@ -2389,7 +2399,8 @@ Label& slow_case // continuation point if fast allocation fails ); void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); - void incr_allocated_bytes(Register var_size_in_bytes, int con_size_in_bytes, Register t1); + void incr_allocated_bytes(RegisterOrConstant size_in_bytes, + Register t1, Register t2); // interface method calling void lookup_interface_method(Register recv_klass,
--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -588,10 +588,13 @@ inline void MacroAssembler::jmp( Register s1, Register s2 ) { jmpl( s1, s2, G0 ); } inline void MacroAssembler::jmp( Register s1, int simm13a, RelocationHolder const& rspec ) { jmpl( s1, simm13a, G0, rspec); } +inline bool MacroAssembler::is_far_target(address d) { + return !is_in_wdisp30_range(d, CodeCache::low_bound()) || !is_in_wdisp30_range(d, CodeCache::high_bound()); +} + // Call with a check to see if we need to deal with the added // expense of relocation and if we overflow the displacement -// of the quick call instruction./ -// Check to see if we have to deal with relocations +// of the quick call instruction. inline void MacroAssembler::call( address d, relocInfo::relocType rt ) { #ifdef _LP64 intptr_t disp; @@ -603,14 +606,12 @@ // Is this address within range of the call instruction? // If not, use the expensive instruction sequence - disp = (intptr_t)d - (intptr_t)pc(); - if ( disp != (intptr_t)(int32_t)disp ) { + if (is_far_target(d)) { relocate(rt); AddressLiteral dest(d); jumpl_to(dest, O7, O7); - } - else { - Assembler::call( d, rt ); + } else { + Assembler::call(d, rt); } #else Assembler::call( d, rt );
--- a/hotspot/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2358,6 +2358,8 @@ op->tmp3()->as_register() == G4 && op->tmp4()->as_register() == O1 && op->klass()->as_register() == G5, "must be"); + + LP64_ONLY( __ signx(op->len()->as_register()); ) if (UseSlowPath || (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
--- a/hotspot/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -170,11 +170,13 @@ Register t2, // temp register Label& slow_case // continuation point if fast allocation fails ) { + RegisterOrConstant size_in_bytes = var_size_in_bytes->is_valid() + ? RegisterOrConstant(var_size_in_bytes) : RegisterOrConstant(con_size_in_bytes); if (UseTLAB) { tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); } else { eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); - incr_allocated_bytes(var_size_in_bytes, con_size_in_bytes, t1); + incr_allocated_bytes(size_in_bytes, t1, t2); } }
--- a/hotspot/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -343,9 +343,10 @@ // returned. restore_live_registers(sasm); - __ restore(); - __ br(Assembler::always, false, Assembler::pt, deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type); - __ delayed()->nop(); + + AddressLiteral dest(deopt_blob->unpack_with_reexecution()); + __ jump_to(dest, O0); + __ delayed()->restore(); __ bind(no_deopt); restore_live_registers(sasm); @@ -461,7 +462,7 @@ // get the instance size __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); - __ incr_allocated_bytes(G1_obj_size, 0, G3_t1); + __ incr_allocated_bytes(G1_obj_size, G3_t1, G4_t2); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -577,7 +578,7 @@ __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size - __ incr_allocated_bytes(G1_arr_size, 0, G3_t1); + __ incr_allocated_bytes(G1_arr_size, G3_t1, O1_t2); __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset);
--- a/hotspot/src/cpu/sparc/vm/interp_masm_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/interp_masm_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1295,16 +1295,13 @@ // Get the method data pointer from the methodOop and set the // specified register to its value. -void InterpreterMacroAssembler::set_method_data_pointer_offset(Register Roff) { +void InterpreterMacroAssembler::set_method_data_pointer() { assert(ProfileInterpreter, "must be profiling interpreter"); Label get_continue; ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr); test_method_data_pointer(get_continue); add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr); - if (Roff != noreg) - // Roff contains a method data index ("mdi"). It defaults to zero. - add(ImethodDataPtr, Roff, ImethodDataPtr); bind(get_continue); } @@ -1315,10 +1312,11 @@ Label zero_continue; // Test MDO to avoid the call if it is NULL. - ld_ptr(Lmethod, methodOopDesc::method_data_offset(), ImethodDataPtr); + ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr); test_method_data_pointer(zero_continue); call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), Lmethod, Lbcp); - set_method_data_pointer_offset(O0); + add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr); + add(ImethodDataPtr, O0, ImethodDataPtr); bind(zero_continue); } @@ -1369,7 +1367,6 @@ } void InterpreterMacroAssembler::test_invocation_counter_for_mdp(Register invocation_count, - Register cur_bcp, Register Rtmp, Label &profile_continue) { assert(ProfileInterpreter, "must be profiling interpreter"); @@ -1400,8 +1397,8 @@ delayed()->nop(); // Build it now. - call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), cur_bcp); - set_method_data_pointer_offset(O0); + call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + set_method_data_pointer_for_bcp(); ba(false, profile_continue); delayed()->nop(); bind(done);
--- a/hotspot/src/cpu/sparc/vm/interp_masm_sparc.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/interp_masm_sparc.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -269,12 +269,11 @@ #ifndef CC_INTERP // Interpreter profiling operations - void set_method_data_pointer() { set_method_data_pointer_offset(noreg); } + void set_method_data_pointer(); void set_method_data_pointer_for_bcp(); - void set_method_data_pointer_offset(Register mdi_reg); void test_method_data_pointer(Label& zero_continue); void verify_method_data_pointer(); - void test_invocation_counter_for_mdp(Register invocation_count, Register cur_bcp, Register Rtmp, Label &profile_continue); + void test_invocation_counter_for_mdp(Register invocation_count, Register Rtmp, Label &profile_continue); void set_mdp_data_at(int constant, Register value); void increment_mdp_data_at(Address counter, Register bumped_count,
--- a/hotspot/src/cpu/sparc/vm/methodHandles_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/methodHandles_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -395,7 +395,7 @@ // // Generate an "entry" field for a method handle. // This determines how the method handle will respond to calls. -void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek, TRAPS) { +void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek) { // Here is the register state during an interpreted call, // as set up by generate_method_handle_interpreter_entry(): // - G5: garbage temp (was MethodHandle.invoke methodOop, unused) @@ -447,8 +447,9 @@ // exception. Since we use a C2I adapter to set up the // interpreter state, arguments are expected in compiler // argument registers. - methodHandle mh(raise_exception_method()); - address c2i_entry = methodOopDesc::make_adapters(mh, CATCH); + assert(raise_exception_method(), "must be set"); + address c2i_entry = raise_exception_method()->get_c2i_entry(); + assert(c2i_entry, "method must be linked"); __ mov(O5_savedSP, SP); // Cut the stack back to where the caller started.
--- a/hotspot/src/cpu/sparc/vm/sparc.ad Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/sparc.ad Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -575,7 +575,11 @@ int MachCallRuntimeNode::ret_addr_offset() { #ifdef _LP64 - return NativeFarCall::instruction_size; // farcall; delay slot + if (MacroAssembler::is_far_target(entry_point())) { + return NativeFarCall::instruction_size; + } else { + return NativeCall::instruction_size; + } #else return NativeCall::instruction_size; // call; delay slot #endif @@ -941,7 +945,7 @@ #endif } -void emit_call_reloc(CodeBuffer &cbuf, intptr_t entry_point, relocInfo::relocType rtype, bool preserve_g2 = false, bool force_far_call = false) { +void emit_call_reloc(CodeBuffer &cbuf, intptr_t entry_point, relocInfo::relocType rtype, bool preserve_g2 = false) { // The method which records debug information at every safepoint // expects the call to be the first instruction in the snippet as // it creates a PcDesc structure which tracks the offset of a call @@ -963,20 +967,7 @@ int startpos = __ offset(); #endif /* ASSERT */ -#ifdef _LP64 - // Calls to the runtime or native may not be reachable from compiled code, - // so we generate the far call sequence on 64 bit sparc. - // This code sequence is relocatable to any address, even on LP64. - if ( force_far_call ) { - __ relocate(rtype); - AddressLiteral dest(entry_point); - __ jumpl_to(dest, O7, O7); - } - else -#endif - { - __ call((address)entry_point, rtype); - } + __ call((address)entry_point, rtype); if (preserve_g2) __ delayed()->mov(G2, L7); else __ delayed()->nop(); @@ -2507,7 +2498,7 @@ // CALL directly to the runtime // The user of this is responsible for ensuring that R_L7 is empty (killed). emit_call_reloc(cbuf, $meth$$method, relocInfo::runtime_call_type, - /*preserve_g2=*/true, /*force far call*/true); + /*preserve_g2=*/true); %} enc_class preserve_SP %{
--- a/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1364,15 +1364,8 @@ // We have decided to profile this method in the interpreter __ bind(profile_method); - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), Lbcp, true); - -#ifdef ASSERT - __ tst(O0); - __ breakpoint_trap(Assembler::notEqual); -#endif - - __ set_method_data_pointer(); - + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); __ ba(false, profile_method_continue); __ delayed()->nop(); }
--- a/hotspot/src/cpu/sparc/vm/templateTable_sparc.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/templateTable_sparc.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1689,7 +1689,7 @@ const Register G4_invoke_ctr = G4; __ increment_backedge_counter(G4_invoke_ctr, G1_scratch); if (ProfileInterpreter) { - __ test_invocation_counter_for_mdp(G4_invoke_ctr, Lbcp, G3_scratch, Lforward); + __ test_invocation_counter_for_mdp(G4_invoke_ctr, G3_scratch, Lforward); if (UseOnStackReplacement) { __ test_backedge_count_for_osr(O2_bumped_count, O0_cur_bcp, G3_scratch); } @@ -3447,7 +3447,8 @@ __ delayed()->nop(); // bump total bytes allocated by this thread - __ incr_allocated_bytes(Roffset, 0, G1_scratch); + // RoldTopValue and RtopAddr are dead, so can use G1 and G3 + __ incr_allocated_bytes(Roffset, G1_scratch, G3_scratch); } if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
--- a/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1641,12 +1641,14 @@ } void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { + Register len = op->len()->as_register(); + LP64_ONLY( __ movslq(len, len); ) + if (UseSlowPath || (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { __ jmp(*op->stub()->entry()); } else { - Register len = op->len()->as_register(); Register tmp1 = op->tmp1()->as_register(); Register tmp2 = op->tmp2()->as_register(); Register tmp3 = op->tmp3()->as_register();
--- a/hotspot/src/cpu/x86/vm/globals_x86.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/globals_x86.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -62,7 +62,7 @@ // due to lack of optimization caused by C++ compiler bugs define_pd_global(intx, StackShadowPages, SOLARIS_ONLY(20) NOT_SOLARIS(6) DEBUG_ONLY(+2)); #else -define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+1)); +define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+5)); #endif // AMD64 define_pd_global(intx, PreInflateSpin, 10);
--- a/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -819,7 +819,7 @@ // Set the method data pointer for the current bcp. void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { assert(ProfileInterpreter, "must be profiling interpreter"); - Label zero_continue; + Label set_mdp; push(rax); push(rbx); @@ -827,21 +827,17 @@ // Test MDO to avoid the call if it is NULL. movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); testptr(rax, rax); - jcc(Assembler::zero, zero_continue); - + jcc(Assembler::zero, set_mdp); // rbx,: method // rsi: bcp call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, rsi); // rax,: mdi - + // mdo is guaranteed to be non-zero here, we checked for it before the call. movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - testptr(rbx, rbx); - jcc(Assembler::zero, zero_continue); addptr(rbx, in_bytes(methodDataOopDesc::data_offset())); - addptr(rbx, rax); - movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx); - - bind(zero_continue); + addptr(rax, rbx); + bind(set_mdp); + movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); pop(rbx); pop(rax); }
--- a/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -855,7 +855,7 @@ // Set the method data pointer for the current bcp. void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { assert(ProfileInterpreter, "must be profiling interpreter"); - Label zero_continue; + Label set_mdp; push(rax); push(rbx); @@ -863,21 +863,17 @@ // Test MDO to avoid the call if it is NULL. movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); testptr(rax, rax); - jcc(Assembler::zero, zero_continue); - + jcc(Assembler::zero, set_mdp); // rbx: method // r13: bcp call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, r13); // rax: mdi - + // mdo is guaranteed to be non-zero here, we checked for it before the call. movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - testptr(rbx, rbx); - jcc(Assembler::zero, zero_continue); addptr(rbx, in_bytes(methodDataOopDesc::data_offset())); - addptr(rbx, rax); - movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx); - - bind(zero_continue); + addptr(rax, rbx); + bind(set_mdp); + movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); pop(rbx); pop(rax); }
--- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -390,7 +390,7 @@ // // Generate an "entry" field for a method handle. // This determines how the method handle will respond to calls. -void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek, TRAPS) { +void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek) { // Here is the register state during an interpreted call, // as set up by generate_method_handle_interpreter_entry(): // - rbx: garbage temp (was MethodHandle.invoke methodOop, unused) @@ -451,8 +451,9 @@ // exception. Since we use a C2I adapter to set up the // interpreter state, arguments are expected in compiler // argument registers. - methodHandle mh(raise_exception_method()); - address c2i_entry = methodOopDesc::make_adapters(mh, CHECK); + assert(raise_exception_method(), "must be set"); + address c2i_entry = raise_exception_method()->get_c2i_entry(); + assert(c2i_entry, "method must be linked"); const Register rdi_pc = rax; __ pop(rdi_pc); // caller PC
--- a/hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1367,15 +1367,9 @@ if (ProfileInterpreter) { // We have decided to profile this method in the interpreter __ bind(profile_method); - - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi, true); - - __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop - __ movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); - __ test_method_data_pointer(rax, profile_method_continue); - __ addptr(rax, in_bytes(methodDataOopDesc::data_offset())); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); + __ get_method(rbx); __ jmp(profile_method_continue); } // Handle overflow of counter and compile method
--- a/hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1383,20 +1383,9 @@ if (ProfileInterpreter) { // We have decided to profile this method in the interpreter __ bind(profile_method); - - __ call_VM(noreg, - CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), - r13, true); - - __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop - __ movptr(rax, Address(rbx, - in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rax); - __ test_method_data_pointer(rax, profile_method_continue); - __ addptr(rax, in_bytes(methodDataOopDesc::data_offset())); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rax); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); + __ get_method(rbx); __ jmp(profile_method_continue); } // Handle overflow of counter and compile method
--- a/hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1665,16 +1665,9 @@ if (ProfileInterpreter) { // Out-of-line code to allocate method data oop. __ bind(profile_method); - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); __ load_unsigned_byte(rbx, Address(rsi, 0)); // restore target bytecode - __ movptr(rcx, Address(rbp, method_offset)); - __ movptr(rcx, Address(rcx, in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx); - __ test_method_data_pointer(rcx, dispatch); - // offset non-null mdp by MDO::data_offset() + IR::profile_method() - __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset())); - __ addptr(rcx, rax); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx); + __ set_method_data_pointer_for_bcp(); __ jmp(dispatch); }
--- a/hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1695,21 +1695,9 @@ if (ProfileInterpreter) { // Out-of-line code to allocate method data oop. __ bind(profile_method); - __ call_VM(noreg, - CAST_FROM_FN_PTR(address, - InterpreterRuntime::profile_method), r13); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); __ load_unsigned_byte(rbx, Address(r13, 0)); // restore target bytecode - __ movptr(rcx, Address(rbp, method_offset)); - __ movptr(rcx, Address(rcx, - in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rcx); - __ test_method_data_pointer(rcx, dispatch); - // offset non-null mdp by MDO::data_offset() + IR::profile_method() - __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset())); - __ addptr(rcx, rax); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rcx); + __ set_method_data_pointer_for_bcp(); __ jmp(dispatch); }
--- a/hotspot/src/os/linux/vm/os_linux.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os/linux/vm/os_linux.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1610,10 +1610,9 @@ const char* os::dll_file_extension() { return ".so"; } -const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - return prop == NULL ? "/tmp" : prop; -} +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. +const char* os::get_temp_directory() { return "/tmp"; } static bool file_exists(const char* filename) { struct stat statbuf;
--- a/hotspot/src/os/solaris/vm/os_solaris.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os/solaris/vm/os_solaris.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1884,10 +1884,9 @@ const char* os::dll_file_extension() { return ".so"; } -const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - return prop == NULL ? "/tmp" : prop; -} +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. +const char* os::get_temp_directory() { return "/tmp"; } static bool file_exists(const char* filename) { struct stat statbuf;
--- a/hotspot/src/os/windows/vm/os_windows.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os/windows/vm/os_windows.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1044,9 +1044,9 @@ return 0; } +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - if (prop != 0) return prop; static char path_buf[MAX_PATH]; if (GetTempPath(MAX_PATH, path_buf)>0) return path_buf;
--- a/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,6 +54,8 @@ inline void Atomic::dec_ptr(volatile intptr_t* dest) { (void)add_ptr(-1, dest); } inline void Atomic::dec_ptr(volatile void* dest) { (void)add_ptr(-1, dest); } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + inline jint Atomic::add (jint add_value, volatile jint* dest) { intptr_t rv; __asm__ volatile(
--- a/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -100,11 +100,6 @@ return exchange_value; } -extern "C" { - // defined in linux_x86.s - jlong _Atomic_cmpxchg_long(jlong, volatile jlong*, jlong, bool); -} - #ifdef AMD64 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } @@ -164,9 +159,9 @@ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); } -#else -//inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } -//inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + +#else // !AMD64 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) { return (intptr_t)Atomic::add((jint)add_value, (volatile jint*)dest); @@ -189,6 +184,12 @@ return (intptr_t)xchg((jint)exchange_value, (volatile jint*)dest); } +extern "C" { + // defined in linux_x86.s + jlong _Atomic_cmpxchg_long(jlong, volatile jlong*, jlong, bool); + void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); +} + inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP()); } @@ -200,6 +201,21 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + _Atomic_move_long(src, &dest); + return dest; +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, dest); +} + #endif // AMD64 #endif // OS_CPU_LINUX_X86_VM_ATOMIC_LINUX_X86_INLINE_HPP
--- a/hotspot/src/os_cpu/linux_x86/vm/linux_x86_32.s Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/linux_x86_32.s Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ # -# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,7 @@ .globl _mmx_Copy_arrayof_conjoint_jshorts .globl _Atomic_cmpxchg_long + .globl _Atomic_move_long .text @@ -653,3 +654,15 @@ popl %ebx ret + + # Support for jlong Atomic::load and Atomic::store. + # void _Atomic_move_long(volatile jlong* src, volatile jlong* dst) + .p2align 4,,15 + .type _Atomic_move_long,@function +_Atomic_move_long: + movl 4(%esp), %eax # src + fildll (%eax) + movl 8(%esp), %eax # dest + fistpll (%eax) + ret +
--- a/hotspot/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_LINUX_X86_VM_ORDERACCESS_LINUX_X86_INLINE_HPP #define OS_CPU_LINUX_X86_VM_ORDERACCESS_LINUX_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -64,11 +65,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -79,11 +80,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -178,7 +179,7 @@ : "0" (v), "r" (p) : "memory"); #else - *p = v; fence(); + release_store(p, v); fence(); #endif // AMD64 }
--- a/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,14 +35,12 @@ inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } -inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } -inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } @@ -54,8 +52,49 @@ inline void Atomic::dec_ptr(volatile intptr_t* dest) { (void)add_ptr(-1, dest); } inline void Atomic::dec_ptr(volatile void* dest) { (void)add_ptr(-1, dest); } + +#ifdef _LP64 + +inline void Atomic::store(jlong store_value, jlong* dest) { *dest = store_value; } +inline void Atomic::store(jlong store_value, volatile jlong* dest) { *dest = store_value; } inline jlong Atomic::load(volatile jlong* src) { return *src; } +#else + +extern "C" void _Atomic_move_long_v8(volatile jlong* src, volatile jlong* dst); +extern "C" void _Atomic_move_long_v9(volatile jlong* src, volatile jlong* dst); + +inline void Atomic_move_long(volatile jlong* src, volatile jlong* dst) { +#ifdef COMPILER2 + // Compiler2 does not support v8, it is used only for v9. + assert (VM_Version::v9_instructions_work(), "only supported on v9"); + _Atomic_move_long_v9(src, dst); +#else + // The branch is cheaper then emulated LDD. + if (VM_Version::v9_instructions_work()) { + _Atomic_move_long_v9(src, dst); + } else { + _Atomic_move_long_v8(src, dst); + } +#endif +} + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + Atomic_move_long(src, &dest); + return dest; +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + Atomic_move_long((volatile jlong*)&store_value, dest); +} + +#endif + #ifdef _GNU_SOURCE inline jint Atomic::add (jint add_value, volatile jint* dest) {
--- a/hotspot/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -77,11 +77,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -92,11 +92,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -120,11 +120,11 @@ inline void OrderAccess::release_store_fence(volatile jbyte* p, jbyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jshort* p, jshort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jint* p, jint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile juint* p, juint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jfloat* p, jfloat v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jdouble* p, jdouble v) { *p = v; fence(); }
--- a/hotspot/src/os_cpu/solaris_sparc/vm/solaris_sparc.il Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/solaris_sparc.il Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 2002, 2005, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -152,6 +152,39 @@ .nonvolatile .end + // Support for jlong Atomic::load and Atomic::store on v8. + // + // void _Atomic_move_long_v8(volatile jlong* src, volatile jlong* dst) + // + // Arguments: + // src: O0 + // dest: O1 + // + // Overwrites O2 and O3 + + .inline _Atomic_move_long_v8,2 + .volatile + ldd [%o0], %o2 + std %o2, [%o1] + .nonvolatile + .end + + // Support for jlong Atomic::load and Atomic::store on v9. + // + // void _Atomic_move_long_v9(volatile jlong* src, volatile jlong* dst) + // + // Arguments: + // src: O0 + // dest: O1 + // + // Overwrites O2 + + .inline _Atomic_move_long_v9,2 + .volatile + ldx [%o0], %o2 + stx %o2, [%o1] + .nonvolatile + .end // Support for jint Atomic::add(jint add_value, volatile jint* dest). //
--- a/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -151,14 +151,22 @@ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } -extern "C" void _Atomic_load_long(volatile jlong* src, volatile jlong* dst); +extern "C" void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); inline jlong Atomic::load(volatile jlong* src) { volatile jlong dest; - _Atomic_load_long(src, &dest); + _Atomic_move_long(src, &dest); return dest; } +inline void Atomic::store(jlong store_value, jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, dest); +} + #endif // AMD64 #ifdef _GNU_SOURCE
--- a/hotspot/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_SOLARIS_X86_VM_ORDERACCESS_SOLARIS_X86_INLINE_HPP #define OS_CPU_SOLARIS_X86_VM_ORDERACCESS_SOLARIS_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -80,11 +81,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -95,11 +96,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -123,11 +124,11 @@ inline void OrderAccess::release_store_fence(volatile jbyte* p, jbyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jshort* p, jshort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jint* p, jint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile juint* p, juint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jfloat* p, jfloat v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jdouble* p, jdouble v) { *p = v; fence(); }
--- a/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -104,8 +104,9 @@ popl %ebx .end - // Support for void Atomic::load(volatile jlong* src, volatile jlong* dest). - .inline _Atomic_load_long,2 + // Support for jlong Atomic::load and Atomic::store. + // void _Atomic_move_long(volatile jlong* src, volatile jlong* dst) + .inline _Atomic_move_long,2 movl 0(%esp), %eax // src fildll (%eax) movl 4(%esp), %eax // dest
--- a/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -137,10 +137,10 @@ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + #else // !AMD64 -//inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } -//inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } inline jint Atomic::add (jint add_value, volatile jint* dest) { int mp = os::is_MP(); __asm { @@ -254,6 +254,33 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + volatile jlong* pdest = &dest; + __asm { + mov eax, src + fild qword ptr [eax] + mov eax, pdest + fistp qword ptr [eax] + } + return dest; +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + volatile jlong* src = &store_value; + __asm { + mov eax, src + fild qword ptr [eax] + mov eax, dest + fistp qword ptr [eax] + } +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + Atomic::store(store_value, (volatile jlong*)dest); +} + #endif // AMD64 #pragma warning(default: 4035) // Enables warnings reporting missing return statement
--- a/hotspot/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_WINDOWS_X86_VM_ORDERACCESS_WINDOWS_X86_INLINE_HPP #define OS_CPU_WINDOWS_X86_VM_ORDERACCESS_WINDOWS_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -65,11 +66,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -80,11 +81,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -188,7 +189,7 @@ #endif // AMD64 } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { release_store_fence((volatile jbyte*)p, (jbyte)v); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { release_store_fence((volatile jshort*)p, (jshort)v); }
--- a/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -319,24 +319,24 @@ case Bytecodes::_tableswitch: { // set block for each case - Bytecode_tableswitch *switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int l = switch_->length(); + Bytecode_tableswitch sw(&s); + int l = sw.length(); for (int i = 0; i < l; i++) { - make_block_at(cur_bci + switch_->dest_offset_at(i), current); + make_block_at(cur_bci + sw.dest_offset_at(i), current); } - make_block_at(cur_bci + switch_->default_offset(), current); + make_block_at(cur_bci + sw.default_offset(), current); current = NULL; break; } case Bytecodes::_lookupswitch: { // set block for each case - Bytecode_lookupswitch *switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int l = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int l = sw.number_of_pairs(); for (int i = 0; i < l; i++) { - make_block_at(cur_bci + switch_->pair_at(i)->offset(), current); + make_block_at(cur_bci + sw.pair_at(i).offset(), current); } - make_block_at(cur_bci + switch_->default_offset(), current); + make_block_at(cur_bci + sw.default_offset(), current); current = NULL; break; } @@ -1275,15 +1275,15 @@ void GraphBuilder::table_switch() { - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(method()->code() + bci()); - const int l = switch_->length(); + Bytecode_tableswitch sw(stream()); + const int l = sw.length(); if (CanonicalizeNodes && l == 1) { // total of 2 successors => use If instead of switch // Note: This code should go into the canonicalizer as soon as it can // can handle canonicalized forms that contain more than one node. - Value key = append(new Constant(new IntConstant(switch_->low_key()))); - BlockBegin* tsux = block_at(bci() + switch_->dest_offset_at(0)); - BlockBegin* fsux = block_at(bci() + switch_->default_offset()); + Value key = append(new Constant(new IntConstant(sw.low_key()))); + BlockBegin* tsux = block_at(bci() + sw.dest_offset_at(0)); + BlockBegin* fsux = block_at(bci() + sw.default_offset()); bool is_bb = tsux->bci() < bci() || fsux->bci() < bci(); ValueStack* state_before = is_bb ? copy_state_before() : NULL; append(new If(ipop(), If::eql, true, key, tsux, fsux, state_before, is_bb)); @@ -1293,29 +1293,29 @@ int i; bool has_bb = false; for (i = 0; i < l; i++) { - sux->at_put(i, block_at(bci() + switch_->dest_offset_at(i))); - if (switch_->dest_offset_at(i) < 0) has_bb = true; + sux->at_put(i, block_at(bci() + sw.dest_offset_at(i))); + if (sw.dest_offset_at(i) < 0) has_bb = true; } // add default successor - sux->at_put(i, block_at(bci() + switch_->default_offset())); + sux->at_put(i, block_at(bci() + sw.default_offset())); ValueStack* state_before = has_bb ? copy_state_before() : NULL; - append(new TableSwitch(ipop(), sux, switch_->low_key(), state_before, has_bb)); + append(new TableSwitch(ipop(), sux, sw.low_key(), state_before, has_bb)); } } void GraphBuilder::lookup_switch() { - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(method()->code() + bci()); - const int l = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(stream()); + const int l = sw.number_of_pairs(); if (CanonicalizeNodes && l == 1) { // total of 2 successors => use If instead of switch // Note: This code should go into the canonicalizer as soon as it can // can handle canonicalized forms that contain more than one node. // simplify to If - LookupswitchPair* pair = switch_->pair_at(0); - Value key = append(new Constant(new IntConstant(pair->match()))); - BlockBegin* tsux = block_at(bci() + pair->offset()); - BlockBegin* fsux = block_at(bci() + switch_->default_offset()); + LookupswitchPair pair = sw.pair_at(0); + Value key = append(new Constant(new IntConstant(pair.match()))); + BlockBegin* tsux = block_at(bci() + pair.offset()); + BlockBegin* fsux = block_at(bci() + sw.default_offset()); bool is_bb = tsux->bci() < bci() || fsux->bci() < bci(); ValueStack* state_before = is_bb ? copy_state_before() : NULL; append(new If(ipop(), If::eql, true, key, tsux, fsux, state_before, is_bb)); @@ -1326,13 +1326,13 @@ int i; bool has_bb = false; for (i = 0; i < l; i++) { - LookupswitchPair* pair = switch_->pair_at(i); - if (pair->offset() < 0) has_bb = true; - sux->at_put(i, block_at(bci() + pair->offset())); - keys->at_put(i, pair->match()); + LookupswitchPair pair = sw.pair_at(i); + if (pair.offset() < 0) has_bb = true; + sux->at_put(i, block_at(bci() + pair.offset())); + keys->at_put(i, pair.match()); } // add default successor - sux->at_put(i, block_at(bci() + switch_->default_offset())); + sux->at_put(i, block_at(bci() + sw.default_offset())); ValueStack* state_before = has_bb ? copy_state_before() : NULL; append(new LookupSwitch(ipop(), sux, keys, state_before, has_bb)); }
--- a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1990,9 +1990,8 @@ LIR_Opr reg = reg = rlock_result(x, x->basic_type()); + get_Object_unsafe(reg, src.result(), off.result(), type, x->is_volatile()); if (x->is_volatile() && os::is_MP()) __ membar_acquire(); - get_Object_unsafe(reg, src.result(), off.result(), type, x->is_volatile()); - if (x->is_volatile() && os::is_MP()) __ membar(); } @@ -2014,6 +2013,7 @@ if (x->is_volatile() && os::is_MP()) __ membar_release(); put_Object_unsafe(src.result(), off.result(), data.result(), type, x->is_volatile()); + if (x->is_volatile() && os::is_MP()) __ membar(); }
--- a/hotspot/src/share/vm/c1/c1_Runtime1.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_Runtime1.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -369,7 +369,7 @@ if (branch_bci != InvocationEntryBci) { // Compute desination bci address pc = method()->code_base() + branch_bci; - Bytecodes::Code branch = Bytecodes::code_at(pc, method()); + Bytecodes::Code branch = Bytecodes::code_at(method(), pc); int offset = 0; switch (branch) { case Bytecodes::_if_icmplt: case Bytecodes::_iflt: @@ -659,14 +659,14 @@ static klassOop resolve_field_return_klass(methodHandle caller, int bci, TRAPS) { - Bytecode_field* field_access = Bytecode_field_at(caller, bci); + Bytecode_field field_access(caller, bci); // This can be static or non-static field access - Bytecodes::Code code = field_access->code(); + Bytecodes::Code code = field_access.code(); // We must load class, initialize class and resolvethe field FieldAccessInfo result; // initialize class if needed constantPoolHandle constants(THREAD, caller->constants()); - LinkResolver::resolve_field(result, constants, field_access->index(), Bytecodes::java_code(code), false, CHECK_NULL); + LinkResolver::resolve_field(result, constants, field_access.index(), Bytecodes::java_code(code), false, CHECK_NULL); return result.klass()(); } @@ -767,7 +767,7 @@ Events::log("patch_code @ " INTPTR_FORMAT , caller_frame.pc()); - Bytecodes::Code code = Bytecode_at(caller_method->bcp_from(bci))->java_code(); + Bytecodes::Code code = caller_method()->java_code_at(bci); #ifndef PRODUCT // this is used by assertions in the access_field_patching_id @@ -779,11 +779,11 @@ Handle load_klass(THREAD, NULL); // oop needed by load_klass_patching code if (stub_id == Runtime1::access_field_patching_id) { - Bytecode_field* field_access = Bytecode_field_at(caller_method, bci); + Bytecode_field field_access(caller_method, bci); FieldAccessInfo result; // initialize class if needed - Bytecodes::Code code = field_access->code(); + Bytecodes::Code code = field_access.code(); constantPoolHandle constants(THREAD, caller_method->constants()); - LinkResolver::resolve_field(result, constants, field_access->index(), Bytecodes::java_code(code), false, CHECK); + LinkResolver::resolve_field(result, constants, field_access.index(), Bytecodes::java_code(code), false, CHECK); patch_field_offset = result.field_offset(); // If we're patching a field which is volatile then at compile it @@ -811,36 +811,36 @@ } break; case Bytecodes::_new: - { Bytecode_new* bnew = Bytecode_new_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(bnew->index(), CHECK); + { Bytecode_new bnew(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(bnew.index(), CHECK); } break; case Bytecodes::_multianewarray: - { Bytecode_multianewarray* mna = Bytecode_multianewarray_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(mna->index(), CHECK); + { Bytecode_multianewarray mna(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(mna.index(), CHECK); } break; case Bytecodes::_instanceof: - { Bytecode_instanceof* io = Bytecode_instanceof_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(io->index(), CHECK); + { Bytecode_instanceof io(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(io.index(), CHECK); } break; case Bytecodes::_checkcast: - { Bytecode_checkcast* cc = Bytecode_checkcast_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(cc->index(), CHECK); + { Bytecode_checkcast cc(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(cc.index(), CHECK); } break; case Bytecodes::_anewarray: - { Bytecode_anewarray* anew = Bytecode_anewarray_at(caller_method->bcp_from(bci)); - klassOop ek = caller_method->constants()->klass_at(anew->index(), CHECK); + { Bytecode_anewarray anew(caller_method(), caller_method->bcp_from(bci)); + klassOop ek = caller_method->constants()->klass_at(anew.index(), CHECK); k = Klass::cast(ek)->array_klass(CHECK); } break; case Bytecodes::_ldc: case Bytecodes::_ldc_w: { - Bytecode_loadconstant* cc = Bytecode_loadconstant_at(caller_method, bci); - k = cc->resolve_constant(CHECK); + Bytecode_loadconstant cc(caller_method, bci); + k = cc.resolve_constant(CHECK); assert(k != NULL && !k->is_klass(), "must be class mirror or other Java constant"); } break;
--- a/hotspot/src/share/vm/ci/bcEscapeAnalyzer.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/bcEscapeAnalyzer.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -761,15 +761,15 @@ case Bytecodes::_tableswitch: { state.spop(); - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int len = switch_->length(); + Bytecode_tableswitch sw(&s); + int len = sw.length(); int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->dest_offset_at(i); + dest_bci = s.cur_bci() + sw.dest_offset_at(i); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); assert(s.next_bci() == limit_bci, "branch must end block"); @@ -779,15 +779,15 @@ case Bytecodes::_lookupswitch: { state.spop(); - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int len = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int len = sw.number_of_pairs(); int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->pair_at(i)->offset(); + dest_bci = s.cur_bci() + sw.pair_at(i).offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); fall_through = false;
--- a/hotspot/src/share/vm/ci/ciEnv.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciEnv.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -409,15 +409,15 @@ } else { fail_type = _unloaded_ciinstance_klass; } - klassOop found_klass; + KlassHandle found_klass; if (!require_local) { - found_klass = - SystemDictionary::find_constrained_instance_or_array_klass(sym, loader, - KILL_COMPILE_ON_FATAL_(fail_type)); + klassOop kls = SystemDictionary::find_constrained_instance_or_array_klass( + sym, loader, KILL_COMPILE_ON_FATAL_(fail_type)); + found_klass = KlassHandle(THREAD, kls); } else { - found_klass = - SystemDictionary::find_instance_or_array_klass(sym, loader, domain, - KILL_COMPILE_ON_FATAL_(fail_type)); + klassOop kls = SystemDictionary::find_instance_or_array_klass( + sym, loader, domain, KILL_COMPILE_ON_FATAL_(fail_type)); + found_klass = KlassHandle(THREAD, kls); } // If we fail to find an array klass, look again for its element type. @@ -444,9 +444,9 @@ } } - if (found_klass != NULL) { + if (found_klass() != NULL) { // Found it. Build a CI handle. - return get_object(found_klass)->as_klass(); + return get_object(found_klass())->as_klass(); } if (require_local) return NULL;
--- a/hotspot/src/share/vm/ci/ciMethod.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethod.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -144,7 +144,7 @@ Bytecodes::Code java_code_at_bci(int bci) { address bcp = code() + bci; - return Bytecodes::java_code_at(bcp); + return Bytecodes::java_code_at(NULL, bcp); } BCEscapeAnalyzer *get_bcea(); ciMethodBlocks *get_method_blocks();
--- a/hotspot/src/share/vm/ci/ciMethodBlocks.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethodBlocks.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -175,15 +175,15 @@ case Bytecodes::_tableswitch : { cur_block->set_control_bci(bci); - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int len = switch_->length(); + Bytecode_tableswitch sw(&s); + int len = sw.length(); ciBlock *dest; int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->dest_offset_at(i); + dest_bci = s.cur_bci() + sw.dest_offset_at(i); dest = make_block_at(dest_bci); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); make_block_at(dest_bci); if (s.next_bci() < limit_bci) { dest = make_block_at(s.next_bci()); @@ -194,15 +194,15 @@ case Bytecodes::_lookupswitch: { cur_block->set_control_bci(bci); - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int len = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int len = sw.number_of_pairs(); ciBlock *dest; int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->pair_at(i)->offset(); + dest_bci = s.cur_bci() + sw.pair_at(i).offset(); dest = make_block_at(dest_bci); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); dest = make_block_at(dest_bci); if (s.next_bci() < limit_bci) { dest = make_block_at(s.next_bci());
--- a/hotspot/src/share/vm/ci/ciMethodHandle.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethodHandle.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,11 +38,12 @@ // Return an adapter for this MethodHandle. ciMethod* ciMethodHandle::get_adapter(bool is_invokedynamic) const { VM_ENTRY_MARK; - Handle h(get_oop()); methodHandle callee(_callee->get_methodOop()); - MethodHandleCompiler mhc(h, callee, is_invokedynamic, THREAD); - methodHandle m = mhc.compile(CHECK_NULL); + // We catch all exceptions here that could happen in the method + // handle compiler and stop the VM. + MethodHandleCompiler mhc(h, callee, is_invokedynamic, CATCH); + methodHandle m = mhc.compile(CATCH); return CURRENT_ENV->get_object(m())->as_method(); }
--- a/hotspot/src/share/vm/ci/ciStreams.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciStreams.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -78,8 +78,8 @@ else { assert(!is_wide(), "must not be a wide instruction"); } } - Bytecode* bytecode() const { return Bytecode_at(_bc_start); } - Bytecode* next_bytecode() const { return Bytecode_at(_pc); } + Bytecode bytecode() const { return Bytecode(this, _bc_start); } + Bytecode next_bytecode() const { return Bytecode(this, _pc); } public: // End-Of-Bytecodes @@ -151,11 +151,11 @@ bool has_cache_index() const { return Bytecodes::uses_cp_cache(cur_bc_raw()); } int get_index_u1() const { - return bytecode()->get_index_u1(cur_bc_raw()); + return bytecode().get_index_u1(cur_bc_raw()); } int get_index_u1_cpcache() const { - return bytecode()->get_index_u1_cpcache(cur_bc_raw()); + return bytecode().get_index_u1_cpcache(cur_bc_raw()); } // Get a byte index following this bytecode. @@ -169,29 +169,29 @@ // Get 2-byte index (byte swapping depending on which bytecode) int get_index_u2(bool is_wide = false) const { - return bytecode()->get_index_u2(cur_bc_raw(), is_wide); + return bytecode().get_index_u2(cur_bc_raw(), is_wide); } // Get 2-byte index in native byte order. (Rewriter::rewrite makes these.) int get_index_u2_cpcache() const { - return bytecode()->get_index_u2_cpcache(cur_bc_raw()); + return bytecode().get_index_u2_cpcache(cur_bc_raw()); } // Get 4-byte index, for invokedynamic. int get_index_u4() const { - return bytecode()->get_index_u4(cur_bc_raw()); + return bytecode().get_index_u4(cur_bc_raw()); } bool has_index_u4() const { - return bytecode()->has_index_u4(cur_bc_raw()); + return bytecode().has_index_u4(cur_bc_raw()); } // Get dimensions byte (multinewarray) int get_dimensions() const { return *(unsigned char*)(_pc-1); } // Sign-extended index byte/short, no widening - int get_constant_u1() const { return bytecode()->get_constant_u1(instruction_size()-1, cur_bc_raw()); } - int get_constant_u2(bool is_wide = false) const { return bytecode()->get_constant_u2(instruction_size()-2, cur_bc_raw(), is_wide); } + int get_constant_u1() const { return bytecode().get_constant_u1(instruction_size()-1, cur_bc_raw()); } + int get_constant_u2(bool is_wide = false) const { return bytecode().get_constant_u2(instruction_size()-2, cur_bc_raw(), is_wide); } // Get a byte signed constant for "iinc". Invalid for other bytecodes. // If prefixed with a wide bytecode, get a wide constant @@ -199,18 +199,18 @@ // 2-byte branch offset from current pc int get_dest() const { - return cur_bci() + bytecode()->get_offset_s2(cur_bc_raw()); + return cur_bci() + bytecode().get_offset_s2(cur_bc_raw()); } // 2-byte branch offset from next pc int next_get_dest() const { assert(_pc < _end, ""); - return next_bci() + next_bytecode()->get_offset_s2(Bytecodes::_ifeq); + return next_bci() + next_bytecode().get_offset_s2(Bytecodes::_ifeq); } // 4-byte branch offset from current pc int get_far_dest() const { - return cur_bci() + bytecode()->get_offset_s4(cur_bc_raw()); + return cur_bci() + bytecode().get_offset_s4(cur_bc_raw()); } // For a lookup or switch table, return target destination @@ -407,4 +407,11 @@ } }; + + +// Implementation for declarations in bytecode.hpp +Bytecode::Bytecode(const ciBytecodeStream* stream, address bcp): _bcp(bcp != NULL ? bcp : stream->cur_bcp()), _code(Bytecodes::code_at(NULL, addr_at(0))) {} +Bytecode_lookupswitch::Bytecode_lookupswitch(const ciBytecodeStream* stream): Bytecode(stream) { verify(); } +Bytecode_tableswitch::Bytecode_tableswitch(const ciBytecodeStream* stream): Bytecode(stream) { verify(); } + #endif // SHARE_VM_CI_CISTREAMS_HPP
--- a/hotspot/src/share/vm/ci/ciTypeFlow.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciTypeFlow.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1698,18 +1698,17 @@ break; case Bytecodes::_tableswitch: { - Bytecode_tableswitch *tableswitch = - Bytecode_tableswitch_at(str->cur_bcp()); + Bytecode_tableswitch tableswitch(str); - int len = tableswitch->length(); + int len = tableswitch.length(); _successors = new (arena) GrowableArray<Block*>(arena, len+1, 0, NULL); - int bci = current_bci + tableswitch->default_offset(); + int bci = current_bci + tableswitch.default_offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() == SWITCH_DEFAULT, ""); _successors->append(block); while (--len >= 0) { - int bci = current_bci + tableswitch->dest_offset_at(len); + int bci = current_bci + tableswitch.dest_offset_at(len); block = analyzer->block_at(bci, jsrs); assert(_successors->length() >= SWITCH_CASES, ""); _successors->append_if_missing(block); @@ -1718,19 +1717,18 @@ } case Bytecodes::_lookupswitch: { - Bytecode_lookupswitch *lookupswitch = - Bytecode_lookupswitch_at(str->cur_bcp()); + Bytecode_lookupswitch lookupswitch(str); - int npairs = lookupswitch->number_of_pairs(); + int npairs = lookupswitch.number_of_pairs(); _successors = new (arena) GrowableArray<Block*>(arena, npairs+1, 0, NULL); - int bci = current_bci + lookupswitch->default_offset(); + int bci = current_bci + lookupswitch.default_offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() == SWITCH_DEFAULT, ""); _successors->append(block); while(--npairs >= 0) { - LookupswitchPair *pair = lookupswitch->pair_at(npairs); - int bci = current_bci + pair->offset(); + LookupswitchPair pair = lookupswitch.pair_at(npairs); + int bci = current_bci + pair.offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() >= SWITCH_CASES, ""); _successors->append_if_missing(block);
--- a/hotspot/src/share/vm/classfile/classLoader.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/classfile/classLoader.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1382,3 +1382,61 @@ } #endif //PRODUCT + +// Please keep following two functions at end of this file. With them placed at top or in middle of the file, +// they could get inlined by agressive compiler, an unknown trick, see bug 6966589. +void PerfClassTraceTime::initialize() { + if (!UsePerfData) return; + + if (_eventp != NULL) { + // increment the event counter + _eventp->inc(); + } + + // stop the current active thread-local timer to measure inclusive time + _prev_active_event = -1; + for (int i=0; i < EVENT_TYPE_COUNT; i++) { + if (_timers[i].is_active()) { + assert(_prev_active_event == -1, "should have only one active timer"); + _prev_active_event = i; + _timers[i].stop(); + } + } + + if (_recursion_counters == NULL || (_recursion_counters[_event_type])++ == 0) { + // start the inclusive timer if not recursively called + _t.start(); + } + + // start thread-local timer of the given event type + if (!_timers[_event_type].is_active()) { + _timers[_event_type].start(); + } +} + +PerfClassTraceTime::~PerfClassTraceTime() { + if (!UsePerfData) return; + + // stop the thread-local timer as the event completes + // and resume the thread-local timer of the event next on the stack + _timers[_event_type].stop(); + jlong selftime = _timers[_event_type].ticks(); + + if (_prev_active_event >= 0) { + _timers[_prev_active_event].start(); + } + + if (_recursion_counters != NULL && --(_recursion_counters[_event_type]) > 0) return; + + // increment the counters only on the leaf call + _t.stop(); + _timep->inc(_t.ticks()); + if (_selftimep != NULL) { + _selftimep->inc(selftime); + } + // add all class loading related event selftime to the accumulated time counter + ClassLoader::perf_accumulated_time()->inc(selftime); + + // reset the timer + _timers[_event_type].reset(); +}
--- a/hotspot/src/share/vm/classfile/classLoader.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/classfile/classLoader.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -356,111 +356,57 @@ // (i.e. only one event type) are active at a time even multiple PerfClassTraceTime // instances have been created as multiple events are happening. class PerfClassTraceTime { - public: - enum { - CLASS_LOAD = 0, - PARSE_CLASS = 1, - CLASS_LINK = 2, - CLASS_VERIFY = 3, - CLASS_CLINIT = 4, - DEFINE_CLASS = 5, - EVENT_TYPE_COUNT = 6 - }; - protected: - // _t tracks time from initialization to destruction of this timer instance - // including time for all other event types, and recursive calls of this type. - // When a timer is called recursively, the elapsedTimer _t would not be used. - elapsedTimer _t; - PerfLongCounter* _timep; - PerfLongCounter* _selftimep; - PerfLongCounter* _eventp; - // pointer to thread-local recursion counter and timer array - // The thread_local timers track cumulative time for specific event types - // exclusive of time for other event types, but including recursive calls - // of the same type. - int* _recursion_counters; - elapsedTimer* _timers; - int _event_type; - int _prev_active_event; - - public: - - inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ - PerfLongCounter* selftimep, /* counter incremented with exclusive time */ - PerfLongCounter* eventp, /* event counter */ - int* recursion_counters, /* thread-local recursion counter array */ - elapsedTimer* timers, /* thread-local timer array */ - int type /* event type */ ) : - _timep(timep), _selftimep(selftimep), _eventp(eventp), _recursion_counters(recursion_counters), _timers(timers), _event_type(type) { - initialize(); - } - - inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ - elapsedTimer* timers, /* thread-local timer array */ - int type /* event type */ ) : - _timep(timep), _selftimep(NULL), _eventp(NULL), _recursion_counters(NULL), _timers(timers), _event_type(type) { - initialize(); - } - - void initialize() { - if (!UsePerfData) return; + public: + enum { + CLASS_LOAD = 0, + PARSE_CLASS = 1, + CLASS_LINK = 2, + CLASS_VERIFY = 3, + CLASS_CLINIT = 4, + DEFINE_CLASS = 5, + EVENT_TYPE_COUNT = 6 + }; + protected: + // _t tracks time from initialization to destruction of this timer instance + // including time for all other event types, and recursive calls of this type. + // When a timer is called recursively, the elapsedTimer _t would not be used. + elapsedTimer _t; + PerfLongCounter* _timep; + PerfLongCounter* _selftimep; + PerfLongCounter* _eventp; + // pointer to thread-local recursion counter and timer array + // The thread_local timers track cumulative time for specific event types + // exclusive of time for other event types, but including recursive calls + // of the same type. + int* _recursion_counters; + elapsedTimer* _timers; + int _event_type; + int _prev_active_event; - if (_eventp != NULL) { - // increment the event counter - _eventp->inc(); - } + public: - // stop the current active thread-local timer to measure inclusive time - _prev_active_event = -1; - for (int i=0; i < EVENT_TYPE_COUNT; i++) { - if (_timers[i].is_active()) { - assert(_prev_active_event == -1, "should have only one active timer"); - _prev_active_event = i; - _timers[i].stop(); - } - } - - if (_recursion_counters == NULL || (_recursion_counters[_event_type])++ == 0) { - // start the inclusive timer if not recursively called - _t.start(); - } - - // start thread-local timer of the given event type - if (!_timers[_event_type].is_active()) { - _timers[_event_type].start(); - } - } + inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ + PerfLongCounter* selftimep, /* counter incremented with exclusive time */ + PerfLongCounter* eventp, /* event counter */ + int* recursion_counters, /* thread-local recursion counter array */ + elapsedTimer* timers, /* thread-local timer array */ + int type /* event type */ ) : + _timep(timep), _selftimep(selftimep), _eventp(eventp), _recursion_counters(recursion_counters), _timers(timers), _event_type(type) { + initialize(); + } - inline void suspend() { _t.stop(); _timers[_event_type].stop(); } - inline void resume() { _t.start(); _timers[_event_type].start(); } - - ~PerfClassTraceTime() { - if (!UsePerfData) return; - - // stop the thread-local timer as the event completes - // and resume the thread-local timer of the event next on the stack - _timers[_event_type].stop(); - jlong selftime = _timers[_event_type].ticks(); - - if (_prev_active_event >= 0) { - _timers[_prev_active_event].start(); - } + inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ + elapsedTimer* timers, /* thread-local timer array */ + int type /* event type */ ) : + _timep(timep), _selftimep(NULL), _eventp(NULL), _recursion_counters(NULL), _timers(timers), _event_type(type) { + initialize(); + } - if (_recursion_counters != NULL && --(_recursion_counters[_event_type]) > 0) return; + inline void suspend() { _t.stop(); _timers[_event_type].stop(); } + inline void resume() { _t.start(); _timers[_event_type].start(); } - // increment the counters only on the leaf call - _t.stop(); - _timep->inc(_t.ticks()); - if (_selftimep != NULL) { - _selftimep->inc(selftime); - } - // add all class loading related event selftime to the accumulated time counter - ClassLoader::perf_accumulated_time()->inc(selftime); - - // reset the timer - _timers[_event_type].reset(); - } + ~PerfClassTraceTime(); + void initialize(); }; - #endif // SHARE_VM_CLASSFILE_CLASSLOADER_HPP
--- a/hotspot/src/share/vm/classfile/systemDictionary.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/classfile/systemDictionary.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -172,6 +172,8 @@ \ template(sun_jkernel_DownloadManager_klass, sun_jkernel_DownloadManager, Opt_Kernel) \ \ + template(sun_misc_PostVMInitHook_klass, sun_misc_PostVMInitHook, Opt) \ + \ /* Preload boxing klasses */ \ template(Boolean_klass, java_lang_Boolean, Pre) \ template(Character_klass, java_lang_Character, Pre) \
--- a/hotspot/src/share/vm/classfile/vmSymbols.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/classfile/vmSymbols.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,6 +111,7 @@ template(sun_jkernel_DownloadManager, "sun/jkernel/DownloadManager") \ template(getBootClassPathEntryForClass_name, "getBootClassPathEntryForClass") \ template(setBootClassLoaderHook_name, "setBootClassLoaderHook") \ + template(sun_misc_PostVMInitHook, "sun/misc/PostVMInitHook") \ \ /* class file format tags */ \ template(tag_source_file, "SourceFile") \
--- a/hotspot/src/share/vm/code/nmethod.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/code/nmethod.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1863,9 +1863,9 @@ #ifndef SHARK if (!method()->is_native()) { SimpleScopeDesc ssd(this, fr.pc()); - Bytecode_invoke* call = Bytecode_invoke_at(ssd.method(), ssd.bci()); - bool has_receiver = call->has_receiver(); - symbolOop signature = call->signature(); + Bytecode_invoke call(ssd.method(), ssd.bci()); + bool has_receiver = call.has_receiver(); + symbolOop signature = call.signature(); fr.oops_compiled_arguments_do(signature, has_receiver, reg_map, f); } #endif // !SHARK @@ -2698,8 +2698,7 @@ } else if (sd->method()->is_native()) { st->print("method is native"); } else { - address bcp = sd->method()->bcp_from(sd->bci()); - Bytecodes::Code bc = Bytecodes::java_code_at(bcp); + Bytecodes::Code bc = sd->method()->java_code_at(sd->bci()); st->print(";*%s", Bytecodes::name(bc)); switch (bc) { case Bytecodes::_invokevirtual: @@ -2707,10 +2706,10 @@ case Bytecodes::_invokestatic: case Bytecodes::_invokeinterface: { - Bytecode_invoke* invoke = Bytecode_invoke_at(sd->method(), sd->bci()); + Bytecode_invoke invoke(sd->method(), sd->bci()); st->print(" "); - if (invoke->name() != NULL) - invoke->name()->print_symbol_on(st); + if (invoke.name() != NULL) + invoke.name()->print_symbol_on(st); else st->print("<UNKNOWN>"); break; @@ -2720,10 +2719,10 @@ case Bytecodes::_getstatic: case Bytecodes::_putstatic: { - Bytecode_field* field = Bytecode_field_at(sd->method(), sd->bci()); + Bytecode_field field(sd->method(), sd->bci()); st->print(" "); - if (field->name() != NULL) - field->name()->print_symbol_on(st); + if (field.name() != NULL) + field.name()->print_symbol_on(st); else st->print("<UNKNOWN>"); }
--- a/hotspot/src/share/vm/compiler/methodLiveness.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/compiler/methodLiveness.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -286,16 +286,15 @@ break; case Bytecodes::_tableswitch: { - Bytecode_tableswitch *tableswitch = - Bytecode_tableswitch_at(bytes.cur_bcp()); + Bytecode_tableswitch tableswitch(&bytes); - int len = tableswitch->length(); + int len = tableswitch.length(); - dest = _block_map->at(bci + tableswitch->default_offset()); + dest = _block_map->at(bci + tableswitch.default_offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); while (--len >= 0) { - dest = _block_map->at(bci + tableswitch->dest_offset_at(len)); + dest = _block_map->at(bci + tableswitch.dest_offset_at(len)); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); } @@ -304,17 +303,16 @@ case Bytecodes::_lookupswitch: { - Bytecode_lookupswitch *lookupswitch = - Bytecode_lookupswitch_at(bytes.cur_bcp()); + Bytecode_lookupswitch lookupswitch(&bytes); - int npairs = lookupswitch->number_of_pairs(); + int npairs = lookupswitch.number_of_pairs(); - dest = _block_map->at(bci + lookupswitch->default_offset()); + dest = _block_map->at(bci + lookupswitch.default_offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); while(--npairs >= 0) { - LookupswitchPair *pair = lookupswitch->pair_at(npairs); - dest = _block_map->at( bci + pair->offset()); + LookupswitchPair pair = lookupswitch.pair_at(npairs); + dest = _block_map->at( bci + pair.offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); }
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -3478,6 +3478,7 @@ assert(_collectorState == InitialMarking, "Wrong collector state"); check_correct_thread_executing(); TraceCMSMemoryManagerStats tms(_collectorState); + ReferenceProcessor* rp = ref_processor(); SpecializationStats::clear(); assert(_restart_addr == NULL, "Control point invariant"); @@ -4978,6 +4979,7 @@ if (should_unload_classes()) { CodeCache::gc_epilogue(); } + JvmtiExport::gc_epilogue(); // If we encountered any (marking stack / work queue) overflow // events during the current CMS cycle, take appropriate @@ -5940,11 +5942,6 @@ } rp->verify_no_references_recorded(); assert(!rp->discovery_enabled(), "should have been disabled"); - - // JVMTI object tagging is based on JNI weak refs. If any of these - // refs were cleared then JVMTI needs to update its maps and - // maybe post ObjectFrees to agents. - JvmtiExport::cms_ref_processing_epilogue(); } #ifndef PRODUCT @@ -6305,6 +6302,7 @@ switch (op) { case CMS_op_checkpointRootsInitial: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); checkpointRootsInitial(true); // asynch if (PrintGC) { _cmsGen->printOccupancy("initial-mark"); @@ -6312,6 +6310,7 @@ break; } case CMS_op_checkpointRootsFinal: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); checkpointRootsFinal(true, // asynch false, // !clear_all_soft_refs false); // !init_mark_was_synchronous @@ -7881,25 +7880,23 @@ } // We need this destructor to reclaim any space at the end -// of the space, which do_blk below may not have added back to -// the free lists. [basically dealing with the "fringe effect"] +// of the space, which do_blk below may not yet have added back to +// the free lists. SweepClosure::~SweepClosure() { assert_lock_strong(_freelistLock); - // this should be treated as the end of a free run if any - // The current free range should be returned to the free lists - // as one coalesced chunk. + assert(_limit >= _sp->bottom() && _limit <= _sp->end(), + "sweep _limit out of bounds"); + // Flush any remaining coterminal free run as a single + // coalesced chunk to the appropriate free list. if (inFreeRange()) { - flushCurFreeChunk(freeFinger(), - pointer_delta(_limit, freeFinger())); - assert(freeFinger() < _limit, "the finger pointeth off base"); + assert(freeFinger() < _limit, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(_limit, freeFinger())); if (CMSTraceSweeper) { - gclog_or_tty->print("destructor:"); - gclog_or_tty->print("Sweep:put_free_blk 0x%x ("SIZE_FORMAT") " - "[coalesced:"SIZE_FORMAT"]\n", - freeFinger(), pointer_delta(_limit, freeFinger()), - lastFreeRangeCoalesced()); - } - } + gclog_or_tty->print("Sweep: last chunk: "); + gclog_or_tty->print("put_free_blk 0x%x ("SIZE_FORMAT") [coalesced:"SIZE_FORMAT"]\n", + freeFinger(), pointer_delta(_limit, freeFinger()), lastFreeRangeCoalesced()); + } + } // else nothing to flush NOT_PRODUCT( if (Verbose && PrintGC) { gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " @@ -7936,9 +7933,8 @@ void SweepClosure::initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists) { if (CMSTraceSweeper) { - gclog_or_tty->print("---- Start free range 0x%x with free block [%d] (%d)\n", - freeFinger, _sp->block_size(freeFinger), - freeRangeInFreeLists); + gclog_or_tty->print("---- Start free range at 0x%x with free block (%d)\n", + freeFinger, freeRangeInFreeLists); } assert(!inFreeRange(), "Trampling existing free range"); set_inFreeRange(true); @@ -7993,21 +7989,36 @@ // may have caused us to coalesce the block ending at the address _limit // with a newly expanded chunk (this happens when _limit was set to the // previous _end of the space), so we may have stepped past _limit; see CR 6977970. - if (addr >= _limit) { // we have swept up to or past the limit, do nothing more + if (addr >= _limit) { // we have swept up to or past the limit: finish up assert(_limit >= _sp->bottom() && _limit <= _sp->end(), "sweep _limit out of bounds"); assert(addr < _sp->end(), "addr out of bounds"); - // help the closure application finish + // Flush any remaining coterminal free run as a single + // coalesced chunk to the appropriate free list. + if (inFreeRange()) { + assert(freeFinger() < _limit, "finger points too high"); + flush_cur_free_chunk(freeFinger(), + pointer_delta(addr, freeFinger())); + if (CMSTraceSweeper) { + gclog_or_tty->print("Sweep: last chunk: "); + gclog_or_tty->print("put_free_blk 0x%x ("SIZE_FORMAT") " + "[coalesced:"SIZE_FORMAT"]\n", + freeFinger(), pointer_delta(addr, freeFinger()), + lastFreeRangeCoalesced()); + } + } + + // help the iterator loop finish return pointer_delta(_sp->end(), addr); } + assert(addr < _limit, "sweep invariant"); - // check if we should yield do_yield_check(addr); if (fc->isFree()) { // Chunk that is already free res = fc->size(); - doAlreadyFreeChunk(fc); + do_already_free_chunk(fc); debug_only(_sp->verifyFreeLists()); assert(res == fc->size(), "Don't expect the size to change"); NOT_PRODUCT( @@ -8017,7 +8028,7 @@ NOT_PRODUCT(_last_fc = fc;) } else if (!_bitMap->isMarked(addr)) { // Chunk is fresh garbage - res = doGarbageChunk(fc); + res = do_garbage_chunk(fc); debug_only(_sp->verifyFreeLists()); NOT_PRODUCT( _numObjectsFreed++; @@ -8025,7 +8036,7 @@ ) } else { // Chunk that is alive. - res = doLiveChunk(fc); + res = do_live_chunk(fc); debug_only(_sp->verifyFreeLists()); NOT_PRODUCT( _numObjectsLive++; @@ -8078,7 +8089,7 @@ // to a free list which may be overpopulated. // -void SweepClosure::doAlreadyFreeChunk(FreeChunk* fc) { +void SweepClosure::do_already_free_chunk(FreeChunk* fc) { size_t size = fc->size(); // Chunks that cannot be coalesced are not in the // free lists. @@ -8094,23 +8105,23 @@ // addr and purported end of this block. _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - // Some chunks cannot be coalesced in under any circumstances. + // Some chunks cannot be coalesced under any circumstances. // See the definition of cantCoalesce(). if (!fc->cantCoalesce()) { // This chunk can potentially be coalesced. if (_sp->adaptive_freelists()) { // All the work is done in - doPostIsFreeOrGarbageChunk(fc, size); + do_post_free_or_garbage_chunk(fc, size); } else { // Not adaptive free lists // this is a free chunk that can potentially be coalesced by the sweeper; if (!inFreeRange()) { // if the next chunk is a free block that can't be coalesced // it doesn't make sense to remove this chunk from the free lists FreeChunk* nextChunk = (FreeChunk*)(addr + size); - assert((HeapWord*)nextChunk <= _limit, "sweep invariant"); - if ((HeapWord*)nextChunk < _limit && // there's a next chunk... - nextChunk->isFree() && // which is free... - nextChunk->cantCoalesce()) { // ... but cant be coalesced + assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?"); + if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ... + nextChunk->isFree() && // ... which is free... + nextChunk->cantCoalesce()) { // ... but can't be coalesced // nothing to do } else { // Potentially the start of a new free range: @@ -8156,14 +8167,14 @@ // as the end of a free run if any if (inFreeRange()) { // we kicked some butt; time to pick up the garbage - assert(freeFinger() < addr, "the finger pointeth off base"); - flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger())); + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); } // else, nothing to do, just continue } } -size_t SweepClosure::doGarbageChunk(FreeChunk* fc) { +size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) { // This is a chunk of garbage. It is not in any free list. // Add it to a free list or let it possibly be coalesced into // a larger chunk. @@ -8175,7 +8186,7 @@ // addr and purported end of just dead object. _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - doPostIsFreeOrGarbageChunk(fc, size); + do_post_free_or_garbage_chunk(fc, size); } else { if (!inFreeRange()) { // start of a new free range @@ -8214,35 +8225,16 @@ return size; } -size_t SweepClosure::doLiveChunk(FreeChunk* fc) { +size_t SweepClosure::do_live_chunk(FreeChunk* fc) { HeapWord* addr = (HeapWord*) fc; // The sweeper has just found a live object. Return any accumulated // left hand chunk to the free lists. if (inFreeRange()) { - if (_sp->adaptive_freelists()) { - flushCurFreeChunk(freeFinger(), - pointer_delta(addr, freeFinger())); - } else { // not adaptive freelists - set_inFreeRange(false); - // Add the free range back to the free list if it is not already - // there. - if (!freeRangeInFreeLists()) { - assert(freeFinger() < addr, "the finger pointeth off base"); - if (CMSTraceSweeper) { - gclog_or_tty->print("Sweep:put_free_blk 0x%x (%d) " - "[coalesced:%d]\n", - freeFinger(), pointer_delta(addr, freeFinger()), - lastFreeRangeCoalesced()); - } - _sp->addChunkAndRepairOffsetTable(freeFinger(), - pointer_delta(addr, freeFinger()), lastFreeRangeCoalesced()); - } - } - } - - // Common code path for original and adaptive free lists. - - // this object is live: we'd normally expect this to be + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); + } + + // This object is live: we'd normally expect this to be // an oop, and like to assert the following: // assert(oop(addr)->is_oop(), "live block should be an oop"); // However, as we commented above, this may be an object whose @@ -8257,7 +8249,7 @@ assert(size == CompactibleFreeListSpace::adjustObjectSize(size), "alignment problem"); - #ifdef DEBUG +#ifdef DEBUG if (oop(addr)->klass_or_null() != NULL && ( !_collector->should_unload_classes() || (oop(addr)->is_parsable()) && @@ -8271,7 +8263,7 @@ CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()), "P-mark and computed size do not agree"); } - #endif +#endif } else { // This should be an initialized object that's alive. @@ -8298,18 +8290,16 @@ return size; } -void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc, - size_t chunkSize) { - // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation - // scheme. +void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc, + size_t chunkSize) { + // do_post_free_or_garbage_chunk() should only be called in the case + // of the adaptive free list allocator. bool fcInFreeLists = fc->isFree(); assert(_sp->adaptive_freelists(), "Should only be used in this case."); assert((HeapWord*)fc <= _limit, "sweep invariant"); if (CMSTestInFreeList && fcInFreeLists) { - assert(_sp->verifyChunkInFreeLists(fc), - "free chunk is not in free lists"); - } - + assert(_sp->verifyChunkInFreeLists(fc), "free chunk is not in free lists"); + } if (CMSTraceSweeper) { gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize); @@ -8382,20 +8372,21 @@ if (inFreeRange()) { // In a free range but cannot coalesce with the right hand chunk. // Put the current free range into the free lists. - flushCurFreeChunk(freeFinger(), - pointer_delta(addr, freeFinger())); + flush_cur_free_chunk(freeFinger(), + pointer_delta(addr, freeFinger())); } // Set up for new free range. Pass along whether the right hand // chunk is in the free lists. initialize_free_range((HeapWord*)fc, fcInFreeLists); } } -void SweepClosure::flushCurFreeChunk(HeapWord* chunk, size_t size) { + +void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) { assert(inFreeRange(), "Should only be called if currently in a free range."); assert(size > 0, "A zero sized chunk cannot be added to the free lists."); if (!freeRangeInFreeLists()) { - if(CMSTestInFreeList) { + if (CMSTestInFreeList) { FreeChunk* fc = (FreeChunk*) chunk; fc->setSize(size); assert(!_sp->verifyChunkInFreeLists(fc), @@ -8430,7 +8421,7 @@ // chunk just flushed, they will need to wait for the next // sweep to be coalesced. if (inFreeRange()) { - flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger())); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); } // First give up the locks, then yield, then re-lock.
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1701,7 +1701,9 @@ CMSCollector* _collector; // collector doing the work ConcurrentMarkSweepGeneration* _g; // Generation being swept CompactibleFreeListSpace* _sp; // Space being swept - HeapWord* _limit; + HeapWord* _limit;// the address at which the sweep should stop because + // we do not expect blocks eligible for sweeping past + // that address. Mutex* _freelistLock; // Free list lock (in space) CMSBitMap* _bitMap; // Marking bit map (in // generation) @@ -1745,14 +1747,13 @@ private: // Code that is common to a free chunk or garbage when // encountered during sweeping. - void doPostIsFreeOrGarbageChunk(FreeChunk *fc, - size_t chunkSize); + void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize); // Process a free chunk during sweeping. - void doAlreadyFreeChunk(FreeChunk *fc); + void do_already_free_chunk(FreeChunk *fc); // Process a garbage chunk during sweeping. - size_t doGarbageChunk(FreeChunk *fc); + size_t do_garbage_chunk(FreeChunk *fc); // Process a live chunk during sweeping. - size_t doLiveChunk(FreeChunk* fc); + size_t do_live_chunk(FreeChunk* fc); // Accessors. HeapWord* freeFinger() const { return _freeFinger; } @@ -1769,7 +1770,7 @@ // Initialize a free range. void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists); // Return this chunk to the free lists. - void flushCurFreeChunk(HeapWord* chunk, size_t size); + void flush_cur_free_chunk(HeapWord* chunk, size_t size); // Check if we should yield and do so when necessary. inline void do_yield_check(HeapWord* addr);
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc_implementation/g1/g1RemSet.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" +#include "gc_implementation/shared/vmGCOperations.hpp" #include "memory/genOopClosures.inline.hpp" #include "memory/referencePolicy.hpp" #include "memory/resourceArea.hpp" @@ -457,6 +458,7 @@ _marking_task_overhead(1.0), _cleanup_sleep_factor(0.0), _cleanup_task_overhead(1.0), + _cleanup_list("Cleanup List"), _region_bm(max_regions, false /* in_resource_area*/), _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >> CardTableModRefBS::card_shift, @@ -520,12 +522,6 @@ SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); satb_qs.set_buffer_size(G1SATBBufferSize); - int size = (int) MAX2(ParallelGCThreads, (size_t)1); - _par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size); - for (int i = 0 ; i < size; i++) { - _par_cleanup_thread_state[i] = new ParCleanupThreadState; - } - _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num); _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num); @@ -710,11 +706,6 @@ } ConcurrentMark::~ConcurrentMark() { - int size = (int) MAX2(ParallelGCThreads, (size_t)1); - for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i]; - FREE_C_HEAP_ARRAY(ParCleanupThreadState*, - _par_cleanup_thread_state); - for (int i = 0; i < (int) _max_task_num; ++i) { delete _task_queues->queue(i); delete _tasks[i]; @@ -1142,6 +1133,8 @@ return; } + SvcGCMarker sgcm(SvcGCMarker::OTHER); + if (VerifyDuringGC) { HandleMark hm; // handle scope gclog_or_tty->print(" VerifyDuringGC:(before)"); @@ -1168,12 +1161,12 @@ if (G1TraceMarkStackOverflow) gclog_or_tty->print_cr("\nRemark led to restart for overflow."); } else { + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); // We're done with marking. // This is the end of the marking cycle, we're expected all // threads to have SATB queues with active set to true. - JavaThread::satb_mark_queue_set().set_active_all_threads( - false, /* new active value */ - true /* expected_active */); + satb_mq_set.set_active_all_threads(false, /* new active value */ + true /* expected_active */); if (VerifyDuringGC) { HandleMark hm; // handle scope @@ -1507,21 +1500,20 @@ size_t _max_live_bytes; size_t _regions_claimed; size_t _freed_bytes; - size_t _cleared_h_regions; - size_t _freed_regions; - UncleanRegionList* _unclean_region_list; + FreeRegionList _local_cleanup_list; + HumongousRegionSet _humongous_proxy_set; double _claimed_region_time; double _max_region_time; public: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - UncleanRegionList* list, int worker_num); size_t freed_bytes() { return _freed_bytes; } - size_t cleared_h_regions() { return _cleared_h_regions; } - size_t freed_regions() { return _freed_regions; } - UncleanRegionList* unclean_region_list() { - return _unclean_region_list; + FreeRegionList* local_cleanup_list() { + return &_local_cleanup_list; + } + HumongousRegionSet* humongous_proxy_set() { + return &_humongous_proxy_set; } bool doHeapRegion(HeapRegion *r); @@ -1534,25 +1526,22 @@ class G1ParNoteEndTask: public AbstractGangTask { friend class G1NoteEndOfConcMarkClosure; + protected: G1CollectedHeap* _g1h; size_t _max_live_bytes; size_t _freed_bytes; - ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state; + FreeRegionList* _cleanup_list; + public: G1ParNoteEndTask(G1CollectedHeap* g1h, - ConcurrentMark::ParCleanupThreadState** - par_cleanup_thread_state) : + FreeRegionList* cleanup_list) : AbstractGangTask("G1 note end"), _g1h(g1h), - _max_live_bytes(0), _freed_bytes(0), - _par_cleanup_thread_state(par_cleanup_thread_state) - {} + _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { } void work(int i) { double start = os::elapsedTime(); - G1NoteEndOfConcMarkClosure g1_note_end(_g1h, - &_par_cleanup_thread_state[i]->list, - i); + G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i); if (G1CollectedHeap::use_parallel_gc_threads()) { _g1h->heap_region_par_iterate_chunked(&g1_note_end, i, HeapRegion::NoteEndClaimValue); @@ -1561,14 +1550,18 @@ } assert(g1_note_end.complete(), "Shouldn't have yielded!"); - // Now finish up freeing the current thread's regions. - _g1h->finish_free_region_work(g1_note_end.freed_bytes(), - g1_note_end.cleared_h_regions(), - 0, NULL); + // Now update the lists + _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), + NULL /* free_list */, + g1_note_end.humongous_proxy_set(), + true /* par */); { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); _max_live_bytes += g1_note_end.max_live_bytes(); _freed_bytes += g1_note_end.freed_bytes(); + + _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list()); + assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition"); } double end = os::elapsedTime(); if (G1PrintParCleanupStats) { @@ -1609,30 +1602,28 @@ G1NoteEndOfConcMarkClosure:: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - UncleanRegionList* list, int worker_num) : _g1(g1), _worker_num(worker_num), _max_live_bytes(0), _regions_claimed(0), - _freed_bytes(0), _cleared_h_regions(0), _freed_regions(0), + _freed_bytes(0), _claimed_region_time(0.0), _max_region_time(0.0), - _unclean_region_list(list) -{} - -bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) { + _local_cleanup_list("Local Cleanup List"), + _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { } + +bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) { // We use a claim value of zero here because all regions // were claimed with value 1 in the FinalCount task. - r->reset_gc_time_stamp(); - if (!r->continuesHumongous()) { + hr->reset_gc_time_stamp(); + if (!hr->continuesHumongous()) { double start = os::elapsedTime(); _regions_claimed++; - r->note_end_of_marking(); - _max_live_bytes += r->max_live_bytes(); - _g1->free_region_if_totally_empty_work(r, - _freed_bytes, - _cleared_h_regions, - _freed_regions, - _unclean_region_list, - true /*par*/); + hr->note_end_of_marking(); + _max_live_bytes += hr->max_live_bytes(); + _g1->free_region_if_totally_empty(hr, + &_freed_bytes, + &_local_cleanup_list, + &_humongous_proxy_set, + true /* par */); double region_time = (os::elapsedTime() - start); _claimed_region_time += region_time; if (region_time > _max_region_time) _max_region_time = region_time; @@ -1652,6 +1643,8 @@ return; } + g1h->verify_region_sets_optional(); + if (VerifyDuringGC) { HandleMark hm; // handle scope gclog_or_tty->print(" VerifyDuringGC:(before)"); @@ -1716,7 +1709,7 @@ // Note end of marking in all heap regions. double note_end_start = os::elapsedTime(); - G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state); + G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list); if (G1CollectedHeap::use_parallel_gc_threads()) { int n_workers = g1h->workers()->total_workers(); g1h->set_par_threads(n_workers); @@ -1728,9 +1721,14 @@ } else { g1_par_note_end_task.work(0); } - g1h->set_unclean_regions_coming(true); + + if (!cleanup_list_is_empty()) { + // The cleanup list is not empty, so we'll have to process it + // concurrently. Notify anyone else that might be wanting free + // regions that there will be more free regions coming soon. + g1h->set_free_regions_coming(); + } double note_end_end = os::elapsedTime(); - // Tell the mutators that there might be unclean regions coming... if (G1PrintParCleanupStats) { gclog_or_tty->print_cr(" note end of marking: %8.3f ms.", (note_end_end - note_end_start)*1000.0); @@ -1796,33 +1794,63 @@ /* silent */ false, /* prev marking */ true); } + + g1h->verify_region_sets_optional(); } void ConcurrentMark::completeCleanup() { - // A full collection intervened. if (has_aborted()) return; - int first = 0; - int last = (int)MAX2(ParallelGCThreads, (size_t)1); - for (int t = 0; t < last; t++) { - UncleanRegionList* list = &_par_cleanup_thread_state[t]->list; - assert(list->well_formed(), "Inv"); - HeapRegion* hd = list->hd(); - while (hd != NULL) { - // Now finish up the other stuff. - hd->rem_set()->clear(); - HeapRegion* next_hd = hd->next_from_unclean_list(); - (void)list->pop(); - assert(list->hd() == next_hd, "how not?"); - _g1h->put_region_on_unclean_list(hd); - if (!hd->isHumongous()) { - // Add this to the _free_regions count by 1. - _g1h->finish_free_region_work(0, 0, 1, NULL); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + _cleanup_list.verify_optional(); + FreeRegionList local_free_list("Local Cleanup List"); + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "cleanup list has "SIZE_FORMAT" entries", + _cleanup_list.length()); + } + + // Noone else should be accessing the _cleanup_list at this point, + // so it's not necessary to take any locks + while (!_cleanup_list.is_empty()) { + HeapRegion* hr = _cleanup_list.remove_head(); + assert(hr != NULL, "the list was not empty"); + hr->rem_set()->clear(); + local_free_list.add_as_tail(hr); + + // Instead of adding one region at a time to the secondary_free_list, + // we accumulate them in the local list and move them a few at a + // time. This also cuts down on the number of notify_all() calls + // we do during this process. We'll also append the local list when + // _cleanup_list is empty (which means we just removed the last + // region from the _cleanup_list). + if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) || + _cleanup_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "appending "SIZE_FORMAT" entries to the " + "secondary_free_list, clean list still has " + SIZE_FORMAT" entries", + local_free_list.length(), + _cleanup_list.length()); } - hd = list->hd(); - assert(hd == next_hd, "how not?"); + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + g1h->secondary_free_list_add_as_tail(&local_free_list); + SecondaryFreeList_lock->notify_all(); + } + + if (G1StressConcRegionFreeing) { + for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { + os::sleep(Thread::current(), (jlong) 1, false); + } + } } } + assert(local_free_list.is_empty(), "post-condition"); } bool G1CMIsAliveClosure::do_object_b(oop obj) { @@ -2894,9 +2922,9 @@ virtual void do_oop( oop* p) { do_oop_work(p); } template <class T> void do_oop_work(T* p) { - assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); - assert(!_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(), - "invariant"); + assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) p)), "invariant"); oop obj = oopDesc::load_decode_heap_oop(p); if (_cm->verbose_high()) @@ -3116,8 +3144,8 @@ void CMTask::push(oop obj) { HeapWord* objAddr = (HeapWord*) obj; assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); - assert(!_g1h->heap_region_containing(objAddr)->is_on_free_list(), - "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); assert(!_g1h->is_obj_ill(obj), "invariant"); assert(_nextMarkBitMap->isMarked(objAddr), "invariant"); @@ -3362,8 +3390,8 @@ (void*) obj); assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); - assert(!_g1h->heap_region_containing(obj)->is_on_free_list(), - "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); scan_object(obj);
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/g1/heapRegionSets.hpp" #include "utilities/taskqueue.hpp" class G1CollectedHeap; @@ -369,13 +369,7 @@ double _cleanup_sleep_factor; double _cleanup_task_overhead; - // Stuff related to age cohort processing. - struct ParCleanupThreadState { - char _pre[64]; - UncleanRegionList list; - char _post[64]; - }; - ParCleanupThreadState** _par_cleanup_thread_state; + FreeRegionList _cleanup_list; // CMS marking support structures CMBitMap _markBitMap1; @@ -484,6 +478,10 @@ // prints all gathered CM-related statistics void print_stats(); + bool cleanup_list_is_empty() { + return _cleanup_list.is_empty(); + } + // accessor methods size_t parallel_marking_threads() { return _parallel_marking_threads; } double sleep_factor() { return _sleep_factor; }
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -95,8 +95,8 @@ _vtime_start = os::elapsedVTime(); wait_for_universe_init(); - G1CollectedHeap* g1 = G1CollectedHeap::heap(); - G1CollectorPolicy* g1_policy = g1->g1_policy(); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1_policy = g1h->g1_policy(); G1MMUTracker *mmu_tracker = g1_policy->mmu_tracker(); Thread *current_thread = Thread::current(); @@ -119,7 +119,7 @@ if (!g1_policy->in_young_gc_mode()) { // this ensures the flag is not set if we bail out of the marking // cycle; normally the flag is cleared immediately after cleanup - g1->set_marking_complete(); + g1h->set_marking_complete(); if (g1_policy->adaptive_young_list_length()) { double now = os::elapsedTime(); @@ -228,10 +228,20 @@ VM_CGC_Operation op(&cl_cl, verbose_str); VMThread::execute(&op); } else { - G1CollectedHeap::heap()->set_marking_complete(); + g1h->set_marking_complete(); } - if (!cm()->has_aborted()) { + // Check if cleanup set the free_regions_coming flag. If it + // hasn't, we can just skip the next step. + if (g1h->free_regions_coming()) { + // The following will finish freeing up any regions that we + // found to be empty during cleanup. We'll do this part + // without joining the suspendible set. If an evacuation pause + // takes places, then we would carry on freeing regions in + // case they are needed by the pause. If a Full GC takes + // places, it would wait for us to process the regions + // reclaimed by cleanup. + double cleanup_start_sec = os::elapsedTime(); if (PrintGC) { gclog_or_tty->date_stamp(PrintGCDateStamps); @@ -240,23 +250,22 @@ } // Now do the remainder of the cleanup operation. - _sts.join(); _cm->completeCleanup(); - if (!cm()->has_aborted()) { - g1_policy->record_concurrent_mark_cleanup_completed(); + g1_policy->record_concurrent_mark_cleanup_completed(); - double cleanup_end_sec = os::elapsedTime(); - if (PrintGC) { - gclog_or_tty->date_stamp(PrintGCDateStamps); - gclog_or_tty->stamp(PrintGCTimeStamps); - gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf]", - cleanup_end_sec - cleanup_start_sec); - } + double cleanup_end_sec = os::elapsedTime(); + if (PrintGC) { + gclog_or_tty->date_stamp(PrintGCDateStamps); + gclog_or_tty->stamp(PrintGCTimeStamps); + gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf]", + cleanup_end_sec - cleanup_start_sec); } - _sts.leave(); + + // We're done: no more free regions coming. + g1h->reset_free_regions_coming(); } - // We're done: no more unclean regions coming. - G1CollectedHeap::heap()->set_unclean_regions_coming(false); + guarantee(cm()->cleanup_list_is_empty(), + "at this point there should be no regions on the cleanup list"); if (cm()->has_aborted()) { if (PrintGC) { @@ -278,7 +287,7 @@ // Java thread is waiting for a full GC to happen (e.g., it // called System.gc() with +ExplicitGCInvokesConcurrent). _sts.join(); - g1->increment_full_collections_completed(true /* concurrent */); + g1h->increment_full_collections_completed(true /* concurrent */); _sts.leave(); } assert(_should_terminate, "just checking");
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp Fri Feb 18 11:15:40 2011 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/space.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/copy.hpp" - -// ======= Concurrent Zero-Fill Thread ======== - -// The CM thread is created when the G1 garbage collector is used - -int ConcurrentZFThread::_region_allocs = 0; -int ConcurrentZFThread::_sync_zfs = 0; -int ConcurrentZFThread::_zf_waits = 0; -int ConcurrentZFThread::_regions_filled = 0; - -ConcurrentZFThread::ConcurrentZFThread() : - ConcurrentGCThread() -{ - create_and_start(); -} - -void ConcurrentZFThread::wait_for_ZF_completed(HeapRegion* hr) { - assert(ZF_mon->owned_by_self(), "Precondition."); - note_zf_wait(); - while (hr->zero_fill_state() == HeapRegion::ZeroFilling) { - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - } -} - -void ConcurrentZFThread::processHeapRegion(HeapRegion* hr) { - assert(!Universe::heap()->is_gc_active(), - "This should not happen during GC."); - assert(hr != NULL, "Precondition"); - // These are unlocked reads, but if this test is successful, then no - // other thread will attempt this zero filling. Only a GC thread can - // modify the ZF state of a region whose state is zero-filling, and this - // should only happen while the ZF thread is locking out GC. - if (hr->zero_fill_state() == HeapRegion::ZeroFilling - && hr->zero_filler() == Thread::current()) { - assert(hr->top() == hr->bottom(), "better be empty!"); - assert(!hr->isHumongous(), "Only free regions on unclean list."); - Copy::fill_to_words(hr->bottom(), hr->capacity()/HeapWordSize); - note_region_filled(); - } -} - -void ConcurrentZFThread::run() { - initialize_in_thread(); - Thread* thr_self = Thread::current(); - _vtime_start = os::elapsedVTime(); - wait_for_universe_init(); - - G1CollectedHeap* g1 = G1CollectedHeap::heap(); - _sts.join(); - while (!_should_terminate) { - _sts.leave(); - - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - - // This local variable will hold a region being zero-filled. This - // region will neither be on the unclean or zero-filled lists, and - // will not be available for allocation; thus, we might have an - // allocation fail, causing a full GC, because of this, but this is a - // price we will pay. (In future, we might want to make the fact - // that there's a region being zero-filled apparent to the G1 heap, - // which could then wait for it in this extreme case...) - HeapRegion* to_fill; - - while (!g1->should_zf() - || (to_fill = g1->pop_unclean_region_list_locked()) == NULL) - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - while (to_fill->zero_fill_state() == HeapRegion::ZeroFilling) - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - - // So now to_fill is non-NULL and is not ZeroFilling. It might be - // Allocated or ZeroFilled. (The latter could happen if this thread - // starts the zero-filling of a region, but a GC intervenes and - // pushes new regions needing on the front of the filling on the - // front of the list.) - - switch (to_fill->zero_fill_state()) { - case HeapRegion::Allocated: - to_fill = NULL; - break; - - case HeapRegion::NotZeroFilled: - to_fill->set_zero_fill_in_progress(thr_self); - - ZF_mon->unlock(); - _sts.join(); - processHeapRegion(to_fill); - _sts.leave(); - ZF_mon->lock_without_safepoint_check(); - - if (to_fill->zero_fill_state() == HeapRegion::ZeroFilling - && to_fill->zero_filler() == thr_self) { - to_fill->set_zero_fill_complete(); - (void)g1->put_free_region_on_list_locked(to_fill); - } - break; - - case HeapRegion::ZeroFilled: - (void)g1->put_free_region_on_list_locked(to_fill); - break; - - case HeapRegion::ZeroFilling: - ShouldNotReachHere(); - break; - } - } - _vtime_accum = (os::elapsedVTime() - _vtime_start); - _sts.join(); - } - _sts.leave(); - - assert(_should_terminate, "just checking"); - terminate(); -} - -bool ConcurrentZFThread::offer_yield() { - if (_sts.should_yield()) { - _sts.yield("Concurrent ZF"); - return true; - } else { - return false; - } -} - -void ConcurrentZFThread::stop() { - // it is ok to take late safepoints here, if needed - MutexLockerEx mu(Terminator_lock); - _should_terminate = true; - while (!_has_terminated) { - Terminator_lock->wait(); - } -} - -void ConcurrentZFThread::print() const { - print_on(tty); -} - -void ConcurrentZFThread::print_on(outputStream* st) const { - st->print("\"G1 Concurrent Zero-Fill Thread\" "); - Thread::print_on(st); - st->cr(); -} - - -double ConcurrentZFThread::_vtime_accum; - -void ConcurrentZFThread::print_summary_info() { - gclog_or_tty->print("\nConcurrent Zero-Filling:\n"); - gclog_or_tty->print(" Filled %d regions, used %5.2fs.\n", - _regions_filled, - vtime_accum()); - gclog_or_tty->print(" Of %d region allocs, %d (%5.2f%%) required sync ZF,\n", - _region_allocs, _sync_zfs, - (_region_allocs > 0 ? - (float)_sync_zfs/(float)_region_allocs*100.0 : - 0.0)); - gclog_or_tty->print(" and %d (%5.2f%%) required a ZF wait.\n", - _zf_waits, - (_region_allocs > 0 ? - (float)_zf_waits/(float)_region_allocs*100.0 : - 0.0)); - -}
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp Fri Feb 18 11:15:40 2011 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP - -#include "gc_implementation/shared/concurrentGCThread.hpp" - -// The Concurrent ZF Thread. Performs concurrent zero-filling. - -class ConcurrentZFThread: public ConcurrentGCThread { - friend class VMStructs; - friend class ZeroFillRegionClosure; - - private: - - // Zero fill the heap region. - void processHeapRegion(HeapRegion* r); - - // Stats - // Allocation (protected by heap lock). - static int _region_allocs; // Number of regions allocated - static int _sync_zfs; // Synchronous zero-fills + - static int _zf_waits; // Wait for conc zero-fill completion. - - // Number of regions CFZ thread fills. - static int _regions_filled; - - double _vtime_start; // Initial virtual time. - - // These are static because the "print_summary_info" method is, and - // it currently assumes there is only one ZF thread. We'll change when - // we need to. - static double _vtime_accum; // Initial virtual time. - static double vtime_accum() { return _vtime_accum; } - - // Offer yield for GC. Returns true if yield occurred. - bool offer_yield(); - - public: - // Constructor - ConcurrentZFThread(); - - // Main loop. - virtual void run(); - - // Printing - void print_on(outputStream* st) const; - void print() const; - - // Waits until "r" has been zero-filled. Requires caller to hold the - // ZF_mon. - static void wait_for_ZF_completed(HeapRegion* r); - - // Get or clear the current unclean region. Should be done - // while holding the ZF_needed_mon lock. - - // shutdown - void stop(); - - // Stats - static void note_region_alloc() {_region_allocs++; } - static void note_sync_zfs() { _sync_zfs++; } - static void note_zf_wait() { _zf_waits++; } - static void note_region_filled() { _regions_filled++; } - - static void print_summary_info(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -222,7 +222,7 @@ // Action_mark - update the BOT for the block [blk_start, blk_end). // Current typical use is for splitting a block. -// Action_single - udpate the BOT for an allocation. +// Action_single - update the BOT for an allocation. // Action_verify - BOT verification. void G1BlockOffsetArray::do_block_internal(HeapWord* blk_start, HeapWord* blk_end, @@ -331,47 +331,6 @@ do_block_internal(blk_start, blk_end, Action_mark); } -void G1BlockOffsetArray::join_blocks(HeapWord* blk1, HeapWord* blk2) { - HeapWord* blk1_start = Universe::heap()->block_start(blk1); - HeapWord* blk2_start = Universe::heap()->block_start(blk2); - assert(blk1 == blk1_start && blk2 == blk2_start, - "Must be block starts."); - assert(blk1 + _sp->block_size(blk1) == blk2, "Must be contiguous."); - size_t blk1_start_index = _array->index_for(blk1); - size_t blk2_start_index = _array->index_for(blk2); - assert(blk1_start_index <= blk2_start_index, "sanity"); - HeapWord* blk2_card_start = _array->address_for_index(blk2_start_index); - if (blk2 == blk2_card_start) { - // blk2 starts a card. Does blk1 start on the prevous card, or futher - // back? - assert(blk1_start_index < blk2_start_index, "must be lower card."); - if (blk1_start_index + 1 == blk2_start_index) { - // previous card; new value for blk2 card is size of blk1. - _array->set_offset_array(blk2_start_index, (u_char) _sp->block_size(blk1)); - } else { - // Earlier card; go back a card. - _array->set_offset_array(blk2_start_index, N_words); - } - } else { - // blk2 does not start a card. Does it cross a card? If not, nothing - // to do. - size_t blk2_end_index = - _array->index_for(blk2 + _sp->block_size(blk2) - 1); - assert(blk2_end_index >= blk2_start_index, "sanity"); - if (blk2_end_index > blk2_start_index) { - // Yes, it crosses a card. The value for the next card must change. - if (blk1_start_index + 1 == blk2_start_index) { - // previous card; new value for second blk2 card is size of blk1. - _array->set_offset_array(blk2_start_index + 1, - (u_char) _sp->block_size(blk1)); - } else { - // Earlier card; go back a card. - _array->set_offset_array(blk2_start_index + 1, N_words); - } - } - } -} - HeapWord* G1BlockOffsetArray::block_start_unsafe(const void* addr) { assert(_bottom <= addr && addr < _end, "addr must be covered by this Array"); @@ -580,15 +539,50 @@ #endif } -void -G1BlockOffsetArray::set_for_starts_humongous(HeapWord* new_end) { - assert(_end == new_end, "_end should have already been updated"); +bool +G1BlockOffsetArray::verify_for_object(HeapWord* obj_start, + size_t word_size) const { + size_t first_card = _array->index_for(obj_start); + size_t last_card = _array->index_for(obj_start + word_size - 1); + if (!_array->is_card_boundary(obj_start)) { + // If the object is not on a card boundary the BOT entry of the + // first card should point to another object so we should not + // check that one. + first_card += 1; + } + for (size_t card = first_card; card <= last_card; card += 1) { + HeapWord* card_addr = _array->address_for_index(card); + HeapWord* block_start = block_start_const(card_addr); + if (block_start != obj_start) { + gclog_or_tty->print_cr("block start: "PTR_FORMAT" is incorrect - " + "card index: "SIZE_FORMAT" " + "card addr: "PTR_FORMAT" BOT entry: %u " + "obj: "PTR_FORMAT" word size: "SIZE_FORMAT" " + "cards: ["SIZE_FORMAT","SIZE_FORMAT"]", + block_start, card, card_addr, + _array->offset_array(card), + obj_start, word_size, first_card, last_card); + return false; + } + } + return true; +} - // The first BOT entry should have offset 0. - _array->set_offset_array(_array->index_for(_bottom), 0); - // The rest should point to the first one. - set_remainder_to_point_to_start(_bottom + N_words, new_end); +#ifndef PRODUCT +void +G1BlockOffsetArray::print_on(outputStream* out) { + size_t from_index = _array->index_for(_bottom); + size_t to_index = _array->index_for(_end); + out->print_cr(">> BOT for area ["PTR_FORMAT","PTR_FORMAT") " + "cards ["SIZE_FORMAT","SIZE_FORMAT")", + _bottom, _end, from_index, to_index); + for (size_t i = from_index; i < to_index; ++i) { + out->print_cr(" entry "SIZE_FORMAT_W(8)" | "PTR_FORMAT" : %3u", + i, _array->address_for_index(i), + (uint) _array->offset_array(i)); + } } +#endif // !PRODUCT ////////////////////////////////////////////////////////////////////// // G1BlockOffsetArrayContigSpace @@ -641,10 +635,20 @@ } void -G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_end) { - G1BlockOffsetArray::set_for_starts_humongous(new_end); +G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_top) { + assert(new_top <= _end, "_end should have already been updated"); + + // The first BOT entry should have offset 0. + zero_bottom_entry(); + initialize_threshold(); + alloc_block(_bottom, new_top); + } - // Make sure _next_offset_threshold and _next_offset_index point to new_end. - _next_offset_threshold = new_end; - _next_offset_index = _array->index_for(new_end); +#ifndef PRODUCT +void +G1BlockOffsetArrayContigSpace::print_on(outputStream* out) { + G1BlockOffsetArray::print_on(out); + out->print_cr(" next offset threshold: "PTR_FORMAT, _next_offset_threshold); + out->print_cr(" next offset index: "SIZE_FORMAT, _next_offset_index); } +#endif // !PRODUCT
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -352,11 +352,6 @@ // The following methods are useful and optimized for a // general, non-contiguous space. - // The given arguments are required to be the starts of adjacent ("blk1" - // before "blk2") well-formed blocks covered by "this". After this call, - // they should be considered to form one block. - virtual void join_blocks(HeapWord* blk1, HeapWord* blk2); - // Given a block [blk_start, blk_start + full_blk_size), and // a left_blk_size < full_blk_size, adjust the BOT to show two // blocks [blk_start, blk_start + left_blk_size) and @@ -429,6 +424,12 @@ verify_single_block(blk, blk + size); } + // Used by region verification. Checks that the contents of the + // BOT reflect that there's a single object that spans the address + // range [obj_start, obj_start + word_size); returns true if this is + // the case, returns false if it's not. + bool verify_for_object(HeapWord* obj_start, size_t word_size) const; + // Verify that the given block is before _unallocated_block inline void verify_not_unallocated(HeapWord* blk_start, HeapWord* blk_end) const { @@ -444,7 +445,7 @@ void check_all_cards(size_t left_card, size_t right_card) const; - virtual void set_for_starts_humongous(HeapWord* new_end); + virtual void print_on(outputStream* out) PRODUCT_RETURN; }; // A subtype of BlockOffsetArray that takes advantage of the fact @@ -494,7 +495,9 @@ HeapWord* block_start_unsafe(const void* addr); HeapWord* block_start_unsafe_const(const void* addr) const; - virtual void set_for_starts_humongous(HeapWord* new_end); + void set_for_starts_humongous(HeapWord* new_top); + + virtual void print_on(outputStream* out) PRODUCT_RETURN; }; #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,6 @@ #include "gc_implementation/g1/concurrentG1Refine.hpp" #include "gc_implementation/g1/concurrentG1RefineThread.hpp" #include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" #include "gc_implementation/g1/g1MarkSweep.hpp" @@ -425,11 +424,9 @@ void G1CollectedHeap::stop_conc_gc_threads() { _cg1r->stop(); - _czft->stop(); _cmThread->stop(); } - void G1CollectedHeap::check_ct_logs_at_safepoint() { DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set(); @@ -481,49 +478,92 @@ // Private methods. -// Finds a HeapRegion that can be used to allocate a given size of block. - - -HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size, - bool do_expand, - bool zero_filled) { - ConcurrentZFThread::note_region_alloc(); - HeapRegion* res = alloc_free_region_from_lists(zero_filled); +HeapRegion* +G1CollectedHeap::new_region_try_secondary_free_list(size_t word_size) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (!_secondary_free_list.is_empty() || free_regions_coming()) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "secondary_free_list has "SIZE_FORMAT" entries", + _secondary_free_list.length()); + } + // It looks as if there are free regions available on the + // secondary_free_list. Let's move them to the free_list and try + // again to allocate from it. + append_secondary_free_list(); + + assert(!_free_list.is_empty(), "if the secondary_free_list was not " + "empty we should have moved at least one entry to the free_list"); + HeapRegion* res = _free_list.remove_head(); + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "allocated "HR_FORMAT" from secondary_free_list", + HR_FORMAT_PARAMS(res)); + } + return res; + } + + // Wait here until we get notifed either when (a) there are no + // more free regions coming or (b) some regions have been moved on + // the secondary_free_list. + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "could not allocate from secondary_free_list"); + } + return NULL; +} + +HeapRegion* G1CollectedHeap::new_region_work(size_t word_size, + bool do_expand) { + assert(!isHumongous(word_size) || + word_size <= (size_t) HeapRegion::GrainWords, + "the only time we use this to allocate a humongous region is " + "when we are allocating a single humongous region"); + + HeapRegion* res; + if (G1StressConcRegionFreeing) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "forced to look at the secondary_free_list"); + } + res = new_region_try_secondary_free_list(word_size); + if (res != NULL) { + return res; + } + } + } + res = _free_list.remove_head_or_null(); + if (res == NULL) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "res == NULL, trying the secondary_free_list"); + } + res = new_region_try_secondary_free_list(word_size); + } if (res == NULL && do_expand) { expand(word_size * HeapWordSize); - res = alloc_free_region_from_lists(zero_filled); - assert(res == NULL || - (!res->isHumongous() && - (!zero_filled || - res->zero_fill_state() == HeapRegion::Allocated)), - "Alloc Regions must be zero filled (and non-H)"); + res = _free_list.remove_head_or_null(); } if (res != NULL) { - if (res->is_empty()) { - _free_regions--; - } - assert(!res->isHumongous() && - (!zero_filled || res->zero_fill_state() == HeapRegion::Allocated), - err_msg("Non-young alloc Regions must be zero filled (and non-H):" - " res->isHumongous()=%d, zero_filled=%d, res->zero_fill_state()=%d", - res->isHumongous(), zero_filled, res->zero_fill_state())); - assert(!res->is_on_unclean_list(), - "Alloc Regions must not be on the unclean list"); if (G1PrintHeapRegions) { - gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], " - "top "PTR_FORMAT, - res->hrs_index(), res->bottom(), res->end(), res->top()); + gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT","PTR_FORMAT"], " + "top "PTR_FORMAT, res->hrs_index(), + res->bottom(), res->end(), res->top()); } } return res; } -HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose, - size_t word_size, - bool zero_filled) { +HeapRegion* G1CollectedHeap::new_gc_alloc_region(int purpose, + size_t word_size) { HeapRegion* alloc_region = NULL; if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) { - alloc_region = newAllocRegion_work(word_size, true, zero_filled); + alloc_region = new_region_work(word_size, true /* do_expand */); if (purpose == GCAllocForSurvived && alloc_region != NULL) { alloc_region->set_survivor(); } @@ -534,81 +574,220 @@ return alloc_region; } +int G1CollectedHeap::humongous_obj_allocate_find_first(size_t num_regions, + size_t word_size) { + int first = -1; + if (num_regions == 1) { + // Only one region to allocate, no need to go through the slower + // path. The caller will attempt the expasion if this fails, so + // let's not try to expand here too. + HeapRegion* hr = new_region_work(word_size, false /* do_expand */); + if (hr != NULL) { + first = hr->hrs_index(); + } else { + first = -1; + } + } else { + // We can't allocate humongous regions while cleanupComplete() is + // running, since some of the regions we find to be empty might not + // yet be added to the free list and it is not straightforward to + // know which list they are on so that we can remove them. Note + // that we only need to do this if we need to allocate more than + // one region to satisfy the current humongous allocation + // request. If we are only allocating one region we use the common + // region allocation code (see above). + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty(); + + if (free_regions() >= num_regions) { + first = _hrs->find_contiguous(num_regions); + if (first != -1) { + for (int i = first; i < first + (int) num_regions; ++i) { + HeapRegion* hr = _hrs->at(i); + assert(hr->is_empty(), "sanity"); + assert(is_on_free_list(hr), "sanity"); + hr->set_pending_removal(true); + } + _free_list.remove_all_pending(num_regions); + } + } + } + return first; +} + // If could fit into free regions w/o expansion, try. // Otherwise, if can expand, do so. // Otherwise, if using ex regions might help, try with ex given back. HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) { - assert_heap_locked_or_at_safepoint(); - assert(regions_accounted_for(), "Region leakage!"); - - // We can't allocate humongous regions while cleanupComplete is - // running, since some of the regions we find to be empty might not - // yet be added to the unclean list. If we're already at a - // safepoint, this call is unnecessary, not to mention wrong. - if (!SafepointSynchronize::is_at_safepoint()) { - wait_for_cleanup_complete(); - } + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); size_t num_regions = round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; - - // Special case if < one region??? - - // Remember the ft size. size_t x_size = expansion_regions(); - - HeapWord* res = NULL; - bool eliminated_allocated_from_lists = false; - - // Can the allocation potentially fit in the free regions? - if (free_regions() >= num_regions) { - res = _hrs->obj_allocate(word_size); - } - if (res == NULL) { - // Try expansion. - size_t fs = _hrs->free_suffix(); + size_t fs = _hrs->free_suffix(); + int first = humongous_obj_allocate_find_first(num_regions, word_size); + if (first == -1) { + // The only thing we can do now is attempt expansion. if (fs + x_size >= num_regions) { expand((num_regions - fs) * HeapRegion::GrainBytes); - res = _hrs->obj_allocate(word_size); - assert(res != NULL, "This should have worked."); - } else { - // Expansion won't help. Are there enough free regions if we get rid - // of reservations? - size_t avail = free_regions(); - if (avail >= num_regions) { - res = _hrs->obj_allocate(word_size); - if (res != NULL) { - remove_allocated_regions_from_lists(); - eliminated_allocated_from_lists = true; - } + first = humongous_obj_allocate_find_first(num_regions, word_size); + assert(first != -1, "this should have worked"); + } + } + + if (first != -1) { + // Index of last region in the series + 1. + int last = first + (int) num_regions; + + // We need to initialize the region(s) we just discovered. This is + // a bit tricky given that it can happen concurrently with + // refinement threads refining cards on these regions and + // potentially wanting to refine the BOT as they are scanning + // those cards (this can happen shortly after a cleanup; see CR + // 6991377). So we have to set up the region(s) carefully and in + // a specific order. + + // The word size sum of all the regions we will allocate. + size_t word_size_sum = num_regions * HeapRegion::GrainWords; + assert(word_size <= word_size_sum, "sanity"); + + // This will be the "starts humongous" region. + HeapRegion* first_hr = _hrs->at(first); + // The header of the new object will be placed at the bottom of + // the first region. + HeapWord* new_obj = first_hr->bottom(); + // This will be the new end of the first region in the series that + // should also match the end of the last region in the seriers. + HeapWord* new_end = new_obj + word_size_sum; + // This will be the new top of the first region that will reflect + // this allocation. + HeapWord* new_top = new_obj + word_size; + + // First, we need to zero the header of the space that we will be + // allocating. When we update top further down, some refinement + // threads might try to scan the region. By zeroing the header we + // ensure that any thread that will try to scan the region will + // come across the zero klass word and bail out. + // + // NOTE: It would not have been correct to have used + // CollectedHeap::fill_with_object() and make the space look like + // an int array. The thread that is doing the allocation will + // later update the object header to a potentially different array + // type and, for a very short period of time, the klass and length + // fields will be inconsistent. This could cause a refinement + // thread to calculate the object size incorrectly. + Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); + + // We will set up the first region as "starts humongous". This + // will also update the BOT covering all the regions to reflect + // that there is a single object that starts at the bottom of the + // first region. + first_hr->set_startsHumongous(new_top, new_end); + + // Then, if there are any, we will set up the "continues + // humongous" regions. + HeapRegion* hr = NULL; + for (int i = first + 1; i < last; ++i) { + hr = _hrs->at(i); + hr->set_continuesHumongous(first_hr); + } + // If we have "continues humongous" regions (hr != NULL), then the + // end of the last one should match new_end. + assert(hr == NULL || hr->end() == new_end, "sanity"); + + // Up to this point no concurrent thread would have been able to + // do any scanning on any region in this series. All the top + // fields still point to bottom, so the intersection between + // [bottom,top] and [card_start,card_end] will be empty. Before we + // update the top fields, we'll do a storestore to make sure that + // no thread sees the update to top before the zeroing of the + // object header and the BOT initialization. + OrderAccess::storestore(); + + // Now that the BOT and the object header have been initialized, + // we can update top of the "starts humongous" region. + assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), + "new_top should be in this region"); + first_hr->set_top(new_top); + + // Now, we will update the top fields of the "continues humongous" + // regions. The reason we need to do this is that, otherwise, + // these regions would look empty and this will confuse parts of + // G1. For example, the code that looks for a consecutive number + // of empty regions will consider them empty and try to + // re-allocate them. We can extend is_empty() to also include + // !continuesHumongous(), but it is easier to just update the top + // fields here. The way we set top for all regions (i.e., top == + // end for all regions but the last one, top == new_top for the + // last one) is actually used when we will free up the humongous + // region in free_humongous_region(). + hr = NULL; + for (int i = first + 1; i < last; ++i) { + hr = _hrs->at(i); + if ((i + 1) == last) { + // last continues humongous region + assert(hr->bottom() < new_top && new_top <= hr->end(), + "new_top should fall on this region"); + hr->set_top(new_top); + } else { + // not last one + assert(new_top > hr->end(), "new_top should be above this region"); + hr->set_top(hr->end()); } } - } - if (res != NULL) { - // Increment by the number of regions allocated. - // FIXME: Assumes regions all of size GrainBytes. -#ifndef PRODUCT - mr_bs()->verify_clean_region(MemRegion(res, res + num_regions * - HeapRegion::GrainWords)); -#endif - if (!eliminated_allocated_from_lists) - remove_allocated_regions_from_lists(); - _summary_bytes_used += word_size * HeapWordSize; - _free_regions -= num_regions; - _num_humongous_regions += (int) num_regions; - } - assert(regions_accounted_for(), "Region Leakage"); - return res; + // If we have continues humongous regions (hr != NULL), then the + // end of the last one should match new_end and its top should + // match new_top. + assert(hr == NULL || + (hr->end() == new_end && hr->top() == new_top), "sanity"); + + assert(first_hr->used() == word_size * HeapWordSize, "invariant"); + _summary_bytes_used += first_hr->used(); + _humongous_set.add(first_hr); + + return new_obj; + } + + verify_region_sets_optional(); + return NULL; } void G1CollectedHeap::retire_cur_alloc_region(HeapRegion* cur_alloc_region) { - // The cleanup operation might update _summary_bytes_used - // concurrently with this method. So, right now, if we don't wait - // for it to complete, updates to _summary_bytes_used might get - // lost. This will be resolved in the near future when the operation - // of the free region list is revamped as part of CR 6977804. - wait_for_cleanup_complete(); + // Other threads might still be trying to allocate using CASes out + // of the region we are retiring, as they can do so without holding + // the Heap_lock. So we first have to make sure that noone else can + // allocate in it by doing a maximal allocation. Even if our CAS + // attempt fails a few times, we'll succeed sooner or later given + // that a failed CAS attempt mean that the region is getting closed + // to being full (someone else succeeded in allocating into it). + size_t free_word_size = cur_alloc_region->free() / HeapWordSize; + + // This is the minimum free chunk we can turn into a dummy + // object. If the free space falls below this, then noone can + // allocate in this region anyway (all allocation requests will be + // of a size larger than this) so we won't have to perform the dummy + // allocation. + size_t min_word_size_to_fill = CollectedHeap::min_fill_size(); + + while (free_word_size >= min_word_size_to_fill) { + HeapWord* dummy = + cur_alloc_region->par_allocate_no_bot_updates(free_word_size); + if (dummy != NULL) { + // If the allocation was successful we should fill in the space. + CollectedHeap::fill_with_object(dummy, free_word_size); + break; + } + + free_word_size = cur_alloc_region->free() / HeapWordSize; + // It's also possible that someone else beats us to the + // allocation and they fill up the region. In that case, we can + // just get out of the loop + } + assert(cur_alloc_region->free() / HeapWordSize < min_word_size_to_fill, + "sanity"); retire_cur_alloc_region_common(cur_alloc_region); assert(_cur_alloc_region == NULL, "post-condition"); @@ -621,7 +800,7 @@ bool at_safepoint, bool do_dirtying, bool can_expand) { - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); assert(_cur_alloc_region == NULL, "replace_cur_alloc_region_and_allocate() should only be called " "after retiring the previous current alloc region"); @@ -632,25 +811,12 @@ "we are not allowed to expand the young gen"); if (can_expand || !g1_policy()->is_young_list_full()) { - if (!at_safepoint) { - // The cleanup operation might update _summary_bytes_used - // concurrently with this method. So, right now, if we don't - // wait for it to complete, updates to _summary_bytes_used might - // get lost. This will be resolved in the near future when the - // operation of the free region list is revamped as part of - // CR 6977804. If we're already at a safepoint, this call is - // unnecessary, not to mention wrong. - wait_for_cleanup_complete(); - } - - HeapRegion* new_cur_alloc_region = newAllocRegion(word_size, - false /* zero_filled */); + HeapRegion* new_cur_alloc_region = new_alloc_region(word_size); if (new_cur_alloc_region != NULL) { assert(new_cur_alloc_region->is_empty(), "the newly-allocated region should be empty, " "as right now we only allocate new regions out of the free list"); g1_policy()->update_region_num(true /* next_is_young */); - _summary_bytes_used -= new_cur_alloc_region->used(); set_region_short_lived_locked(new_cur_alloc_region); assert(!new_cur_alloc_region->isHumongous(), @@ -661,27 +827,29 @@ // young type. OrderAccess::storestore(); - // Now allocate out of the new current alloc region. We could - // have re-used allocate_from_cur_alloc_region() but its - // operation is slightly different to what we need here. First, - // allocate_from_cur_alloc_region() is only called outside a - // safepoint and will always unlock the Heap_lock if it returns - // a non-NULL result. Second, it assumes that the current alloc - // region is what's already assigned in _cur_alloc_region. What - // we want here is to actually do the allocation first before we - // assign the new region to _cur_alloc_region. This ordering is - // not currently important, but it will be essential when we - // change the code to support CAS allocation in the future (see - // CR 6994297). - // - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = new_cur_alloc_region->allocate(word_size); + // Now, perform the allocation out of the region we just + // allocated. Note that noone else can access that region at + // this point (as _cur_alloc_region has not been updated yet), + // so we can just go ahead and do the allocation without any + // atomics (and we expect this allocation attempt to + // suceeded). Given that other threads can attempt an allocation + // with a CAS and without needing the Heap_lock, if we assigned + // the new region to _cur_alloc_region before first allocating + // into it other threads might have filled up the new region + // before we got a chance to do the allocation ourselves. In + // that case, we would have needed to retire the region, grab a + // new one, and go through all this again. Allocating out of the + // new region before assigning it to _cur_alloc_region avoids + // all this. + HeapWord* result = + new_cur_alloc_region->allocate_no_bot_updates(word_size); assert(result != NULL, "we just allocate out of an empty region " "so allocation should have been successful"); assert(is_in(result), "result should be in the heap"); + // Now make sure that the store to _cur_alloc_region does not + // float above the store to top. + OrderAccess::storestore(); _cur_alloc_region = new_cur_alloc_region; if (!at_safepoint) { @@ -698,7 +866,7 @@ assert(_cur_alloc_region == NULL, "we failed to allocate a new current " "alloc region, it should still be NULL"); - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); return NULL; } @@ -710,6 +878,10 @@ assert(!isHumongous(word_size), "attempt_allocation_slow() should not be " "used for humongous allocations"); + // We should only reach here when we were unable to allocate + // otherwise. So, we should have not active current alloc region. + assert(_cur_alloc_region == NULL, "current alloc region should be NULL"); + // We will loop while succeeded is false, which means that we tried // to do a collection, but the VM op did not succeed. So, when we // exit the loop, either one of the allocation attempts was @@ -718,28 +890,8 @@ for (int try_count = 1; /* we'll return or break */; try_count += 1) { bool succeeded = true; - { - // We may have concurrent cleanup working at the time. Wait for - // it to complete. In the future we would probably want to make - // the concurrent cleanup truly concurrent by decoupling it from - // the allocation. This will happen in the near future as part - // of CR 6977804 which will revamp the operation of the free - // region list. The fact that wait_for_cleanup_complete() will - // do a wait() means that we'll give up the Heap_lock. So, it's - // possible that when we exit wait_for_cleanup_complete() we - // might be able to allocate successfully (since somebody else - // might have done a collection meanwhile). So, we'll attempt to - // allocate again, just in case. When we make cleanup truly - // concurrent with allocation, we should remove this allocation - // attempt as it's redundant (we only reach here after an - // allocation attempt has been unsuccessful). - wait_for_cleanup_complete(); - HeapWord* result = attempt_allocation(word_size); - if (result != NULL) { - assert_heap_not_locked(); - return result; - } - } + // Every time we go round the loop we should be holding the Heap_lock. + assert_heap_locked(); if (GC_locker::is_active_and_needs_gc()) { // We are locked out of GC because of the GC locker. We can @@ -748,7 +900,6 @@ if (g1_policy()->can_expand_young_list()) { // Yes, we are allowed to expand the young gen. Let's try to // allocate a new current alloc region. - HeapWord* result = replace_cur_alloc_region_and_allocate(word_size, false, /* at_safepoint */ @@ -771,20 +922,23 @@ // rather than causing more, now probably unnecessary, GC attempts. JavaThread* jthr = JavaThread::current(); assert(jthr != NULL, "sanity"); - if (!jthr->in_critical()) { - MutexUnlocker mul(Heap_lock); - GC_locker::stall_until_clear(); - - // We'll then fall off the end of the ("if GC locker active") - // if-statement and retry the allocation further down in the - // loop. - } else { + if (jthr->in_critical()) { if (CheckJNICalls) { fatal("Possible deadlock due to allocating while" " in jni critical section"); } + // We are returning NULL so the protocol is that we're still + // holding the Heap_lock. + assert_heap_locked(); return NULL; } + + Heap_lock->unlock(); + GC_locker::stall_until_clear(); + + // No need to relock the Heap_lock. We'll fall off to the code + // below the else-statement which assumes that we are not + // holding the Heap_lock. } else { // We are not locked out. So, let's try to do a GC. The VM op // will retry the allocation before it completes. @@ -805,11 +959,10 @@ dirty_young_block(result, word_size); return result; } - - Heap_lock->lock(); } - assert_heap_locked(); + // Both paths that get us here from above unlock the Heap_lock. + assert_heap_not_locked(); // We can reach here when we were unsuccessful in doing a GC, // because another thread beat us to it, or because we were locked @@ -854,7 +1007,7 @@ // allocation paths that attempt to allocate a humongous object // should eventually reach here. Currently, the only paths are from // mem_allocate() and attempt_allocation_at_safepoint(). - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); assert(isHumongous(word_size), "attempt_allocation_humongous() " "should only be used for humongous allocations"); assert(SafepointSynchronize::is_at_safepoint() == at_safepoint, @@ -931,13 +1084,13 @@ } } - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); return NULL; } HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, bool expect_null_cur_alloc_region) { - assert_at_safepoint(); + assert_at_safepoint(true /* should_be_vm_thread */); assert(_cur_alloc_region == NULL || !expect_null_cur_alloc_region, err_msg("the current alloc region was unexpectedly found " "to be non-NULL, cur alloc region: "PTR_FORMAT" " @@ -948,10 +1101,8 @@ if (!expect_null_cur_alloc_region) { HeapRegion* cur_alloc_region = _cur_alloc_region; if (cur_alloc_region != NULL) { - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = cur_alloc_region->allocate(word_size); + // We are at a safepoint so no reason to use the MT-safe version. + HeapWord* result = cur_alloc_region->allocate_no_bot_updates(word_size); if (result != NULL) { assert(is_in(result), "result should be in the heap"); @@ -983,20 +1134,17 @@ assert_heap_not_locked_and_not_at_safepoint(); assert(!isHumongous(word_size), "we do not allow TLABs of humongous size"); - Heap_lock->lock(); - - // First attempt: try allocating out of the current alloc region or - // after replacing the current alloc region. + // First attempt: Try allocating out of the current alloc region + // using a CAS. If that fails, take the Heap_lock and retry the + // allocation, potentially replacing the current alloc region. HeapWord* result = attempt_allocation(word_size); if (result != NULL) { assert_heap_not_locked(); return result; } - assert_heap_locked(); - - // Second attempt: go into the even slower path where we might - // try to schedule a collection. + // Second attempt: Go to the slower path where we might try to + // schedule a collection. result = attempt_allocation_slow(word_size); if (result != NULL) { assert_heap_not_locked(); @@ -1004,6 +1152,7 @@ } assert_heap_locked(); + // Need to unlock the Heap_lock before returning. Heap_lock->unlock(); return NULL; } @@ -1022,11 +1171,10 @@ for (int try_count = 1; /* we'll return */; try_count += 1) { unsigned int gc_count_before; { - Heap_lock->lock(); - if (!isHumongous(word_size)) { - // First attempt: try allocating out of the current alloc - // region or after replacing the current alloc region. + // First attempt: Try allocating out of the current alloc region + // using a CAS. If that fails, take the Heap_lock and retry the + // allocation, potentially replacing the current alloc region. HeapWord* result = attempt_allocation(word_size); if (result != NULL) { assert_heap_not_locked(); @@ -1035,14 +1183,17 @@ assert_heap_locked(); - // Second attempt: go into the even slower path where we might - // try to schedule a collection. + // Second attempt: Go to the slower path where we might try to + // schedule a collection. result = attempt_allocation_slow(word_size); if (result != NULL) { assert_heap_not_locked(); return result; } } else { + // attempt_allocation_humongous() requires the Heap_lock to be held. + Heap_lock->lock(); + HeapWord* result = attempt_allocation_humongous(word_size, false /* at_safepoint */); if (result != NULL) { @@ -1054,7 +1205,8 @@ assert_heap_locked(); // Read the gc count while the heap lock is held. gc_count_before = SharedHeap::heap()->total_collections(); - // We cannot be at a safepoint, so it is safe to unlock the Heap_lock + + // Release the Heap_lock before attempting the collection. Heap_lock->unlock(); } @@ -1092,22 +1244,18 @@ } void G1CollectedHeap::abandon_cur_alloc_region() { - if (_cur_alloc_region != NULL) { - // We're finished with the _cur_alloc_region. - if (_cur_alloc_region->is_empty()) { - _free_regions++; - free_region(_cur_alloc_region); - } else { - // As we're builing (at least the young portion) of the collection - // set incrementally we'll add the current allocation region to - // the collection set here. - if (_cur_alloc_region->is_young()) { - g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region); - } - _summary_bytes_used += _cur_alloc_region->used(); - } - _cur_alloc_region = NULL; - } + assert_at_safepoint(true /* should_be_vm_thread */); + + HeapRegion* cur_alloc_region = _cur_alloc_region; + if (cur_alloc_region != NULL) { + assert(!cur_alloc_region->is_empty(), + "the current alloc region can never be empty"); + assert(cur_alloc_region->is_young(), + "the current alloc region should be young"); + + retire_cur_alloc_region_common(cur_alloc_region); + } + assert(_cur_alloc_region == NULL, "post-condition"); } void G1CollectedHeap::abandon_gc_alloc_regions() { @@ -1188,19 +1336,20 @@ bool G1CollectedHeap::do_collection(bool explicit_gc, bool clear_all_soft_refs, size_t word_size) { + assert_at_safepoint(true /* should_be_vm_thread */); + if (GC_locker::check_active_before_gc()) { return false; } - DTraceGCProbeMarker gc_probe_marker(true /* full */); + SvcGCMarker sgcm(SvcGCMarker::FULL); ResourceMark rm; if (PrintHeapAtGC) { Universe::print_heap_before_gc(); } - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); + verify_region_sets_optional(); const bool do_clear_all_soft_refs = clear_all_soft_refs || collector_policy()->should_clear_all_soft_refs(); @@ -1223,6 +1372,9 @@ double start = os::elapsedTime(); g1_policy()->record_full_collection_start(); + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty(); + gc_prologue(true); increment_total_collections(true /* full gc */); @@ -1235,7 +1387,6 @@ gclog_or_tty->print(" VerifyBeforeGC:"); Universe::verify(true); } - assert(regions_accounted_for(), "Region leakage!"); COMPILER2_PRESENT(DerivedPointerTable::clear()); @@ -1259,7 +1410,6 @@ assert(_cur_alloc_region == NULL, "Invariant."); g1_rem_set()->cleanupHRRS(); tear_down_region_lists(); - set_used_regions_to_need_zero_fill(); // We may have added regions to the current incremental collection // set between the last GC or pause and now. We need to clear the @@ -1294,9 +1444,7 @@ HandleMark hm; // Discard invalid handles created during gc G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs); } - // Because freeing humongous regions may have added some unclean - // regions, it is necessary to tear down again before rebuilding. - tear_down_region_lists(); + assert(free_regions() == 0, "we should not have added any free regions"); rebuild_region_lists(); _summary_bytes_used = recalculate_used(); @@ -1378,7 +1526,6 @@ JavaThread::dirty_card_queue_set().abandon_logs(); assert(!G1DeferredRSUpdate || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any"); - assert(regions_accounted_for(), "Region leakage!"); } if (g1_policy()->in_young_gc_mode()) { @@ -1392,6 +1539,8 @@ // Update the number of full collections that have been completed. increment_full_collections_completed(false /* concurrent */); + verify_region_sets_optional(); + if (PrintHeapAtGC) { Universe::print_heap_after_gc(); } @@ -1532,10 +1681,7 @@ HeapWord* G1CollectedHeap::satisfy_failed_allocation(size_t word_size, bool* succeeded) { - assert(SafepointSynchronize::is_at_safepoint(), - "satisfy_failed_allocation() should only be called at a safepoint"); - assert(Thread::current()->is_VM_thread(), - "satisfy_failed_allocation() should only be called by the VM thread"); + assert_at_safepoint(true /* should_be_vm_thread */); *succeeded = true; // Let's attempt the allocation first. @@ -1607,53 +1753,22 @@ // allocated block, or else "NULL". HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) { - assert(SafepointSynchronize::is_at_safepoint(), - "expand_and_allocate() should only be called at a safepoint"); - assert(Thread::current()->is_VM_thread(), - "expand_and_allocate() should only be called by the VM thread"); + assert_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); size_t expand_bytes = word_size * HeapWordSize; if (expand_bytes < MinHeapDeltaBytes) { expand_bytes = MinHeapDeltaBytes; } expand(expand_bytes); - assert(regions_accounted_for(), "Region leakage!"); + + verify_region_sets_optional(); return attempt_allocation_at_safepoint(word_size, false /* expect_null_cur_alloc_region */); } -size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) { - size_t pre_used = 0; - size_t cleared_h_regions = 0; - size_t freed_regions = 0; - UncleanRegionList local_list; - free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions, - freed_regions, &local_list); - - finish_free_region_work(pre_used, cleared_h_regions, freed_regions, - &local_list); - return pre_used; -} - -void -G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h, - size_t& freed_regions, - UncleanRegionList* list, - bool par) { - assert(!hr->continuesHumongous(), "should have filtered these out"); - size_t res = 0; - if (hr->used() > 0 && hr->garbage_bytes() == hr->used() && - !hr->is_young()) { - if (G1PolicyVerbose > 0) - gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" - " during cleanup", hr, hr->used()); - free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); - } -} - // FIXME: both this and shrink could probably be more efficient by // doing one "VirtualSpace::expand_by" call rather than several. void G1CollectedHeap::expand(size_t expand_bytes) { @@ -1686,19 +1801,7 @@ // Add it to the HeapRegionSeq. _hrs->insert(hr); - // Set the zero-fill state, according to whether it's already - // zeroed. - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - if (is_zeroed) { - hr->set_zero_fill_complete(); - put_free_region_on_list_locked(hr); - } else { - hr->set_zero_fill_needed(); - put_region_on_unclean_list_locked(hr); - } - } - _free_regions++; + _free_list.add_as_tail(hr); // And we used up an expansion region to create it. _expansion_regions--; // Tell the cardtable about it. @@ -1707,6 +1810,7 @@ _bot_shared->resize(_g1_committed.word_size()); } } + if (Verbose && PrintGC) { size_t new_mem_size = _g1_storage.committed_size(); gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK", @@ -1731,7 +1835,6 @@ assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!"); _g1_committed.set_end(mr.start()); - _free_regions -= num_regions_deleted; _expansion_regions += num_regions_deleted; // Tell the cardtable about it. @@ -1751,10 +1854,17 @@ } void G1CollectedHeap::shrink(size_t shrink_bytes) { + verify_region_sets_optional(); + release_gc_alloc_regions(true /* totally */); + // Instead of tearing down / rebuilding the free lists here, we + // could instead use the remove_all_pending() method on free_list to + // remove only the ones that we need to remove. tear_down_region_lists(); // We will rebuild them in a moment. shrink_helper(shrink_bytes); rebuild_region_lists(); + + verify_region_sets_optional(); } // Public methods. @@ -1773,18 +1883,17 @@ _ref_processor(NULL), _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)), _bot_shared(NULL), - _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"), _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL), _evac_failure_scan_stack(NULL) , _mark_in_progress(false), - _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), + _cg1r(NULL), _summary_bytes_used(0), _cur_alloc_region(NULL), _refine_cte_cl(NULL), - _free_region_list(NULL), _free_region_list_size(0), - _free_regions(0), _full_collection(false), - _unclean_region_list(), - _unclean_regions_coming(false), + _free_list("Master Free List"), + _secondary_free_list("Secondary Free List"), + _humongous_set("Master Humongous Set"), + _free_regions_coming(false), _young_list(new YoungList(this)), _gc_time_stamp(0), _surviving_young_words(NULL), @@ -1868,7 +1977,7 @@ ReservedSpace heap_rs(max_byte_size + pgs->max_size(), HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); if (UseCompressedOops) { if (addr != NULL && !heap_rs.is_reserved()) { @@ -1877,13 +1986,13 @@ // Try again to reserver heap higher. addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop); ReservedSpace heap_rs0(total_reserved, HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); if (addr != NULL && !heap_rs0.is_reserved()) { // Failed to reserve at specified address again - give up. addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop); assert(addr == NULL, ""); ReservedSpace heap_rs1(total_reserved, HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); heap_rs = heap_rs1; } else { heap_rs = heap_rs0; @@ -1905,8 +2014,6 @@ _expansion_regions = max_byte_size/HeapRegion::GrainBytes; - _num_humongous_regions = 0; - // Create the gen rem set (and barrier set) for the entire reserved region. _rem_set = collector_policy()->create_rem_set(_reserved, 2); set_barrier_set(rem_set()->bs()); @@ -1951,6 +2058,8 @@ guarantee((size_t) HeapRegion::CardsPerRegion < max_cards_per_region, "too many cards per region"); + HeapRegionSet::set_unrealistically_long_length(max_regions() + 1); + _bot_shared = new G1BlockOffsetSharedArray(_reserved, heap_word_size(init_byte_size)); @@ -1975,11 +2084,6 @@ _cm = new ConcurrentMark(heap_rs, (int) max_regions()); _cmThread = _cm->cmThread(); - // ...and the concurrent zero-fill thread, if necessary. - if (G1ConcZeroFill) { - _czft = new ConcurrentZFThread(); - } - // Initialize the from_card cache structure of HeapRegionRemSet. HeapRegionRemSet::init_heap(max_regions()); @@ -2153,7 +2257,7 @@ #endif // PRODUCT size_t G1CollectedHeap::unsafe_max_alloc() { - if (_free_regions > 0) return HeapRegion::GrainBytes; + if (free_regions() > 0) return HeapRegion::GrainBytes; // otherwise, is there space in the current allocation region? // We need to store the current allocation region in a local variable @@ -2233,8 +2337,7 @@ } void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { - assert(Thread::current()->is_VM_thread(), "Precondition#1"); - assert(Heap_lock->is_locked(), "Precondition#2"); + assert_at_safepoint(true /* should_be_vm_thread */); GCCauseSetter gcs(this, cause); switch (cause) { case GCCause::_heap_inspection: @@ -2257,12 +2360,6 @@ { MutexLocker ml(Heap_lock); - // Don't want to do a GC until cleanup is completed. This - // limitation will be removed in the near future when the - // operation of the free region list is revamped as part of - // CR 6977804. - wait_for_cleanup_complete(); - // Read the GC count while holding the Heap_lock gc_count_before = SharedHeap::heap()->total_collections(); full_gc_count_before = SharedHeap::heap()->total_full_collections(); @@ -2641,10 +2738,6 @@ } } -bool G1CollectedHeap::allocs_are_zero_filled() { - return false; -} - size_t G1CollectedHeap::large_typearray_limit() { // FIXME return HeapRegion::GrainBytes/HeapWordSize; @@ -2659,7 +2752,6 @@ return 0; } - void G1CollectedHeap::prepare_for_verify() { if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { ensure_parsability(false); @@ -2870,7 +2962,9 @@ &rootsCl); bool failures = rootsCl.failures(); rem_set()->invalidate(perm_gen()->used_region(), false); - if (!silent) { gclog_or_tty->print("heapRegions "); } + if (!silent) { gclog_or_tty->print("HeapRegionSets "); } + verify_region_sets(); + if (!silent) { gclog_or_tty->print("HeapRegions "); } if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity check"); @@ -2898,7 +2992,7 @@ failures = true; } } - if (!silent) gclog_or_tty->print("remset "); + if (!silent) gclog_or_tty->print("RemSet "); rem_set()->verify(); if (failures) { @@ -2969,15 +3063,10 @@ if (G1CollectedHeap::use_parallel_gc_threads()) { workers()->print_worker_threads_on(st); } - _cmThread->print_on(st); st->cr(); - _cm->print_worker_threads_on(st); - _cg1r->print_worker_threads_on(st); - - _czft->print_on(st); st->cr(); } @@ -2987,7 +3076,6 @@ } tc->do_thread(_cmThread); _cg1r->threads_do(tc); - tc->do_thread(_czft); } void G1CollectedHeap::print_tracing_info() const { @@ -3003,15 +3091,10 @@ if (G1SummarizeConcMark) { concurrent_mark()->print_summary_info(); } - if (G1SummarizeZFStats) { - ConcurrentZFThread::print_summary_info(); - } g1_policy()->print_yg_surv_rate_info(); - SpecializationStats::print(); } - int G1CollectedHeap::addr_to_arena_id(void* addr) const { HeapRegion* hr = heap_region_containing(addr); if (hr == NULL) { @@ -3210,17 +3293,22 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) { + assert_at_safepoint(true /* should_be_vm_thread */); + guarantee(!is_gc_active(), "collection is not reentrant"); + if (GC_locker::check_active_before_gc()) { return false; } - DTraceGCProbeMarker gc_probe_marker(false /* full */); + SvcGCMarker sgcm(SvcGCMarker::MINOR); ResourceMark rm; if (PrintHeapAtGC) { Universe::print_heap_before_gc(); } + verify_region_sets_optional(); + { // This call will decide whether this pause is an initial-mark // pause. If it is, during_initial_mark_pause() will return true @@ -3251,10 +3339,16 @@ TraceMemoryManagerStats tms(false /* fullGC */); - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); - guarantee(!is_gc_active(), "collection is not reentrant"); - assert(regions_accounted_for(), "Region leakage!"); + // If there are any free regions available on the secondary_free_list + // make sure we append them to the free_list. However, we don't + // have to wait for the rest of the cleanup operation to + // finish. If it's still going on that's OK. If we run out of + // regions, the region allocation code will check the + // secondary_free_list and potentially wait if more free regions + // are coming (see new_region_try_secondary_free_list()). + if (!G1StressConcRegionFreeing) { + append_secondary_free_list_if_not_empty(); + } increment_gc_time_stamp(); @@ -3334,8 +3428,6 @@ // progress, this will be zero. _cm->set_oops_do_bound(); - assert(regions_accounted_for(), "Region leakage."); - if (mark_in_progress()) concurrent_mark()->newCSet(); @@ -3431,8 +3523,6 @@ g1_policy()->record_pause_time_ms(pause_time_ms); g1_policy()->record_collection_pause_end(); - assert(regions_accounted_for(), "Region leakage."); - MemoryService::track_memory_usage(); if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { @@ -3463,8 +3553,6 @@ gc_epilogue(false); } - assert(verify_region_lists(), "Bad region lists."); - if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); print_tracing_info(); @@ -3472,6 +3560,8 @@ } } + verify_region_sets_optional(); + TASKQUEUE_STATS_ONLY(if (ParallelGCVerbose) print_taskqueue_stats()); TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); @@ -3578,7 +3668,7 @@ void G1CollectedHeap::push_gc_alloc_region(HeapRegion* hr) { assert(Thread::current()->is_VM_thread() || - par_alloc_during_gc_lock()->owned_by_self(), "Precondition"); + FreeList_lock->owned_by_self(), "Precondition"); assert(!hr->is_gc_alloc_region() && !hr->in_collection_set(), "Precondition."); hr->set_is_gc_alloc_region(true); @@ -3600,7 +3690,7 @@ #endif // G1_DEBUG void G1CollectedHeap::forget_alloc_region_list() { - assert(Thread::current()->is_VM_thread(), "Precondition"); + assert_at_safepoint(true /* should_be_vm_thread */); while (_gc_alloc_region_list != NULL) { HeapRegion* r = _gc_alloc_region_list; assert(r->is_gc_alloc_region(), "Invariant."); @@ -3620,9 +3710,6 @@ _young_list->add_survivor_region(r); } } - if (r->is_empty()) { - ++_free_regions; - } } #ifdef G1_DEBUG FindGCAllocRegion fa; @@ -3675,7 +3762,7 @@ if (alloc_region == NULL) { // we will get a new GC alloc region - alloc_region = newAllocRegionWithExpansion(ap, 0); + alloc_region = new_gc_alloc_region(ap, 0); } else { // the region was retained from the last collection ++_gc_alloc_region_counts[ap]; @@ -3730,11 +3817,9 @@ set_gc_alloc_region(ap, NULL); if (r->is_empty()) { - // we didn't actually allocate anything in it; let's just put - // it on the free list - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - r->set_zero_fill_complete(); - put_free_region_on_list_locked(r); + // We didn't actually allocate anything in it; let's just put + // it back on the free list. + _free_list.add_as_tail(r); } else if (_retain_gc_alloc_region[ap] && !totally) { // retain it so that we can use it at the beginning of the next GC _retained_gc_alloc_regions[ap] = r; @@ -3856,13 +3941,15 @@ size_t _next_marked_bytes; OopsInHeapRegionClosure *_cl; public: - RemoveSelfPointerClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* cl) : - _g1(g1), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), + RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr, + OopsInHeapRegionClosure* cl) : + _g1(g1), _hr(hr), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), _next_marked_bytes(0), _cl(cl) {} size_t prev_marked_bytes() { return _prev_marked_bytes; } size_t next_marked_bytes() { return _next_marked_bytes; } + // <original comment> // The original idea here was to coalesce evacuated and dead objects. // However that caused complications with the block offset table (BOT). // In particular if there were two TLABs, one of them partially refined. @@ -3871,15 +3958,24 @@ // of TLAB_2. If the last object of the TLAB_1 and the first object // of TLAB_2 are coalesced, then the cards of the unrefined part // would point into middle of the filler object. + // The current approach is to not coalesce and leave the BOT contents intact. + // </original comment> // - // The current approach is to not coalesce and leave the BOT contents intact. + // We now reset the BOT when we start the object iteration over the + // region and refine its entries for every object we come across. So + // the above comment is not really relevant and we should be able + // to coalesce dead objects if we want to. void do_object(oop obj) { + HeapWord* obj_addr = (HeapWord*) obj; + assert(_hr->is_in(obj_addr), "sanity"); + size_t obj_size = obj->size(); + _hr->update_bot_for_object(obj_addr, obj_size); if (obj->is_forwarded() && obj->forwardee() == obj) { // The object failed to move. assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); _cm->markPrev(obj); assert(_cm->isPrevMarked(obj), "Should be marked!"); - _prev_marked_bytes += (obj->size() * HeapWordSize); + _prev_marked_bytes += (obj_size * HeapWordSize); if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { _cm->markAndGrayObjectIfNecessary(obj); } @@ -3901,7 +3997,7 @@ } else { // The object has been either evacuated or is dead. Fill it with a // dummy object. - MemRegion mr((HeapWord*)obj, obj->size()); + MemRegion mr((HeapWord*)obj, obj_size); CollectedHeap::fill_with_object(mr); _cm->clearRangeBothMaps(mr); } @@ -3921,10 +4017,13 @@ HeapRegion* cur = g1_policy()->collection_set(); while (cur != NULL) { assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); - - RemoveSelfPointerClosure rspc(_g1h, cl); + assert(!cur->isHumongous(), "sanity"); + if (cur->evacuation_failed()) { assert(cur->in_collection_set(), "bad CS"); + RemoveSelfPointerClosure rspc(_g1h, cur, cl); + + cur->reset_bot(); cl->set_region(cur); cur->object_iterate(&rspc); @@ -3989,15 +4088,6 @@ } } -void G1CollectedHeap::handle_evacuation_failure(oop old) { - markOop m = old->mark(); - // forward to self - assert(!old->is_forwarded(), "precondition"); - - old->forward_to(old); - handle_evacuation_failure_common(old, m); -} - oop G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop old) { @@ -4084,8 +4174,6 @@ HeapWord* block = alloc_region->par_allocate(word_size); if (block == NULL) { - MutexLockerEx x(par_alloc_during_gc_lock(), - Mutex::_no_safepoint_check_flag); block = allocate_during_gc_slow(purpose, alloc_region, true, word_size); } return block; @@ -4114,6 +4202,12 @@ err_msg("we should not be seeing humongous allocation requests " "during GC, word_size = "SIZE_FORMAT, word_size)); + // We need to make sure we serialize calls to this method. Given + // that the FreeList_lock guards accesses to the free_list anyway, + // and we need to potentially remove a region from it, we'll use it + // to protect the whole call. + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + HeapWord* block = NULL; // In the parallel case, a previous thread to obtain the lock may have // already assigned a new gc_alloc_region. @@ -4159,7 +4253,7 @@ } // Now allocate a new region for allocation. - alloc_region = newAllocRegionWithExpansion(purpose, word_size, false /*zero_filled*/); + alloc_region = new_gc_alloc_region(purpose, word_size); // let the caller handle alloc failure if (alloc_region != NULL) { @@ -4167,9 +4261,6 @@ assert(check_gc_alloc_regions(), "alloc regions messed up"); assert(alloc_region->saved_mark_at_top(), "Mark should have been saved already."); - // We used to assert that the region was zero-filled here, but no - // longer. - // This must be done last: once it's installed, other regions may // allocate in it (without holding the lock.) set_gc_alloc_region(purpose, alloc_region); @@ -4834,91 +4925,91 @@ COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); } -void G1CollectedHeap::free_region(HeapRegion* hr) { - size_t pre_used = 0; - size_t cleared_h_regions = 0; - size_t freed_regions = 0; - UncleanRegionList local_list; - - HeapWord* start = hr->bottom(); - HeapWord* end = hr->prev_top_at_mark_start(); - size_t used_bytes = hr->used(); - size_t live_bytes = hr->max_live_bytes(); - if (used_bytes > 0) { - guarantee( live_bytes <= used_bytes, "invariant" ); - } else { - guarantee( live_bytes == 0, "invariant" ); - } - - size_t garbage_bytes = used_bytes - live_bytes; - if (garbage_bytes > 0) - g1_policy()->decrease_known_garbage_bytes(garbage_bytes); - - free_region_work(hr, pre_used, cleared_h_regions, freed_regions, - &local_list); - finish_free_region_work(pre_used, cleared_h_regions, freed_regions, - &local_list); -} - -void -G1CollectedHeap::free_region_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h_regions, - size_t& freed_regions, - UncleanRegionList* list, - bool par) { - pre_used += hr->used(); - if (hr->isHumongous()) { - assert(hr->startsHumongous(), - "Only the start of a humongous region should be freed."); - int ind = _hrs->find(hr); - assert(ind != -1, "Should have an index."); - // Clear the start region. - hr->hr_clear(par, true /*clear_space*/); - list->insert_before_head(hr); - cleared_h_regions++; - freed_regions++; - // Clear any continued regions. - ind++; - while ((size_t)ind < n_regions()) { - HeapRegion* hrc = _hrs->at(ind); - if (!hrc->continuesHumongous()) break; - // Otherwise, does continue the H region. - assert(hrc->humongous_start_region() == hr, "Huh?"); - hrc->hr_clear(par, true /*clear_space*/); - cleared_h_regions++; - freed_regions++; - list->insert_before_head(hrc); - ind++; +void G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { + if (hr->isHumongous()) { + assert(hr->startsHumongous(), "we should only see starts humongous"); + free_humongous_region(hr, pre_used, free_list, humongous_proxy_set, par); + } else { + free_region(hr, pre_used, free_list, par); } - } else { - hr->hr_clear(par, true /*clear_space*/); - list->insert_before_head(hr); - freed_regions++; - // If we're using clear2, this should not be enabled. - // assert(!hr->in_cohort(), "Can't be both free and in a cohort."); } } -void G1CollectedHeap::finish_free_region_work(size_t pre_used, - size_t cleared_h_regions, - size_t freed_regions, - UncleanRegionList* list) { - if (list != NULL && list->sz() > 0) { - prepend_region_list_on_unclean_list(list); - } - // Acquire a lock, if we're parallel, to update possibly-shared - // variables. - Mutex* lock = (n_par_threads() > 0) ? ParGCRareEvent_lock : NULL; - { +void G1CollectedHeap::free_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + bool par) { + assert(!hr->isHumongous(), "this is only for non-humongous regions"); + assert(!hr->is_empty(), "the region should not be empty"); + assert(free_list != NULL, "pre-condition"); + + *pre_used += hr->used(); + hr->hr_clear(par, true /* clear_space */); + free_list->add_as_tail(hr); +} + +void G1CollectedHeap::free_humongous_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + assert(hr->startsHumongous(), "this is only for starts humongous regions"); + assert(free_list != NULL, "pre-condition"); + assert(humongous_proxy_set != NULL, "pre-condition"); + + size_t hr_used = hr->used(); + size_t hr_capacity = hr->capacity(); + size_t hr_pre_used = 0; + _humongous_set.remove_with_proxy(hr, humongous_proxy_set); + hr->set_notHumongous(); + free_region(hr, &hr_pre_used, free_list, par); + + int i = hr->hrs_index() + 1; + size_t num = 1; + while ((size_t) i < n_regions()) { + HeapRegion* curr_hr = _hrs->at(i); + if (!curr_hr->continuesHumongous()) { + break; + } + curr_hr->set_notHumongous(); + free_region(curr_hr, &hr_pre_used, free_list, par); + num += 1; + i += 1; + } + assert(hr_pre_used == hr_used, + err_msg("hr_pre_used: "SIZE_FORMAT" and hr_used: "SIZE_FORMAT" " + "should be the same", hr_pre_used, hr_used)); + *pre_used += hr_pre_used; +} + +void G1CollectedHeap::update_sets_after_freeing_regions(size_t pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + if (pre_used > 0) { + Mutex* lock = (par) ? ParGCRareEvent_lock : NULL; MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); + assert(_summary_bytes_used >= pre_used, + err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" " + "should be >= pre_used: "SIZE_FORMAT, + _summary_bytes_used, pre_used)); _summary_bytes_used -= pre_used; - _num_humongous_regions -= (int) cleared_h_regions; - _free_regions += freed_regions; + } + if (free_list != NULL && !free_list->is_empty()) { + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + _free_list.add_as_tail(free_list); + } + if (humongous_proxy_set != NULL && !humongous_proxy_set->is_empty()) { + MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); + _humongous_set.update_from_proxy(humongous_proxy_set); } } - void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRegion* list) { while (list != NULL) { guarantee( list->is_young(), "invariant" ); @@ -5041,6 +5132,9 @@ } void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) { + size_t pre_used = 0; + FreeRegionList local_free_list("Local List for CSet Freeing"); + double young_time_ms = 0.0; double non_young_time_ms = 0.0; @@ -5059,6 +5153,8 @@ size_t rs_lengths = 0; while (cur != NULL) { + assert(!is_on_free_list(cur), "sanity"); + if (non_young) { if (cur->is_young()) { double end_sec = os::elapsedTime(); @@ -5069,14 +5165,12 @@ non_young = false; } } else { - if (!cur->is_on_free_list()) { - double end_sec = os::elapsedTime(); - double elapsed_ms = (end_sec - start_sec) * 1000.0; - young_time_ms += elapsed_ms; - - start_sec = os::elapsedTime(); - non_young = true; - } + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = true; } rs_lengths += cur->rem_set()->occupied(); @@ -5109,9 +5203,8 @@ if (!cur->evacuation_failed()) { // And the region is empty. - assert(!cur->is_empty(), - "Should not have empty regions in a CS."); - free_region(cur); + assert(!cur->is_empty(), "Should not have empty regions in a CS."); + free_region(cur, &pre_used, &local_free_list, false /* par */); } else { cur->uninstall_surv_rate_group(); if (cur->is_young()) @@ -5132,6 +5225,9 @@ else young_time_ms += elapsed_ms; + update_sets_after_freeing_regions(pre_used, &local_free_list, + NULL /* humongous_proxy_set */, + false /* par */); policy->record_young_free_cset_time_ms(young_time_ms); policy->record_non_young_free_cset_time_ms(non_young_time_ms); } @@ -5157,291 +5253,53 @@ } } -HeapRegion* -G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) { - assert(ZF_mon->owned_by_self(), "Precondition"); - HeapRegion* res = pop_unclean_region_list_locked(); - if (res != NULL) { - assert(!res->continuesHumongous() && - res->zero_fill_state() != HeapRegion::Allocated, - "Only free regions on unclean list."); - if (zero_filled) { - res->ensure_zero_filled_locked(); - res->set_zero_fill_allocated(); - } - } - return res; -} - -HeapRegion* G1CollectedHeap::alloc_region_from_unclean_list(bool zero_filled) { - MutexLockerEx zx(ZF_mon, Mutex::_no_safepoint_check_flag); - return alloc_region_from_unclean_list_locked(zero_filled); +void G1CollectedHeap::set_free_regions_coming() { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "setting free regions coming"); + } + + assert(!free_regions_coming(), "pre-condition"); + _free_regions_coming = true; } -void G1CollectedHeap::put_region_on_unclean_list(HeapRegion* r) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - put_region_on_unclean_list_locked(r); - if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. -} - -void G1CollectedHeap::set_unclean_regions_coming(bool b) { - MutexLockerEx x(Cleanup_mon); - set_unclean_regions_coming_locked(b); -} - -void G1CollectedHeap::set_unclean_regions_coming_locked(bool b) { - assert(Cleanup_mon->owned_by_self(), "Precondition"); - _unclean_regions_coming = b; - // Wake up mutator threads that might be waiting for completeCleanup to - // finish. - if (!b) Cleanup_mon->notify_all(); -} - -void G1CollectedHeap::wait_for_cleanup_complete() { - assert_not_at_safepoint(); - MutexLockerEx x(Cleanup_mon); - wait_for_cleanup_complete_locked(); -} - -void G1CollectedHeap::wait_for_cleanup_complete_locked() { - assert(Cleanup_mon->owned_by_self(), "precondition"); - while (_unclean_regions_coming) { - Cleanup_mon->wait(); +void G1CollectedHeap::reset_free_regions_coming() { + { + assert(free_regions_coming(), "pre-condition"); + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _free_regions_coming = false; + SecondaryFreeList_lock->notify_all(); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "reset free regions coming"); } } -void -G1CollectedHeap::put_region_on_unclean_list_locked(HeapRegion* r) { - assert(ZF_mon->owned_by_self(), "precondition."); -#ifdef ASSERT - if (r->is_gc_alloc_region()) { - ResourceMark rm; - stringStream region_str; - print_on(®ion_str); - assert(!r->is_gc_alloc_region(), err_msg("Unexpected GC allocation region: %s", - region_str.as_string())); - } -#endif - _unclean_region_list.insert_before_head(r); -} - -void -G1CollectedHeap::prepend_region_list_on_unclean_list(UncleanRegionList* list) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - prepend_region_list_on_unclean_list_locked(list); - if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. -} - -void -G1CollectedHeap:: -prepend_region_list_on_unclean_list_locked(UncleanRegionList* list) { - assert(ZF_mon->owned_by_self(), "precondition."); - _unclean_region_list.prepend_list(list); -} - -HeapRegion* G1CollectedHeap::pop_unclean_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - HeapRegion* res = _unclean_region_list.pop(); - if (res != NULL) { - // Inform ZF thread that there's a new unclean head. - if (_unclean_region_list.hd() != NULL && should_zf()) - ZF_mon->notify_all(); - } - return res; -} - -HeapRegion* G1CollectedHeap::peek_unclean_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - return _unclean_region_list.hd(); -} - - -bool G1CollectedHeap::move_cleaned_region_to_free_list_locked() { - assert(ZF_mon->owned_by_self(), "Precondition"); - HeapRegion* r = peek_unclean_region_list_locked(); - if (r != NULL && r->zero_fill_state() == HeapRegion::ZeroFilled) { - // Result of below must be equal to "r", since we hold the lock. - (void)pop_unclean_region_list_locked(); - put_free_region_on_list_locked(r); - return true; - } else { - return false; - } -} - -bool G1CollectedHeap::move_cleaned_region_to_free_list() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - return move_cleaned_region_to_free_list_locked(); -} - - -void G1CollectedHeap::put_free_region_on_list_locked(HeapRegion* r) { - assert(ZF_mon->owned_by_self(), "precondition."); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - assert(r->zero_fill_state() == HeapRegion::ZeroFilled, - "Regions on free list must be zero filled"); - assert(!r->isHumongous(), "Must not be humongous."); - assert(r->is_empty(), "Better be empty"); - assert(!r->is_on_free_list(), - "Better not already be on free list"); - assert(!r->is_on_unclean_list(), - "Better not already be on unclean list"); - r->set_on_free_list(true); - r->set_next_on_free_list(_free_region_list); - _free_region_list = r; - _free_region_list_size++; - assert(_free_region_list_size == free_region_list_length(), "Inv"); -} - -void G1CollectedHeap::put_free_region_on_list(HeapRegion* r) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - put_free_region_on_list_locked(r); -} - -HeapRegion* G1CollectedHeap::pop_free_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - HeapRegion* res = _free_region_list; - if (res != NULL) { - _free_region_list = res->next_from_free_list(); - _free_region_list_size--; - res->set_on_free_list(false); - res->set_next_on_free_list(NULL); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - } - return res; -} - - -HeapRegion* G1CollectedHeap::alloc_free_region_from_lists(bool zero_filled) { - // By self, or on behalf of self. - assert(Heap_lock->is_locked(), "Precondition"); - HeapRegion* res = NULL; - bool first = true; - while (res == NULL) { - if (zero_filled || !first) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - res = pop_free_region_list_locked(); - if (res != NULL) { - assert(!res->zero_fill_is_allocated(), - "No allocated regions on free list."); - res->set_zero_fill_allocated(); - } else if (!first) { - break; // We tried both, time to return NULL. - } - } - - if (res == NULL) { - res = alloc_region_from_unclean_list(zero_filled); - } - assert(res == NULL || - !zero_filled || - res->zero_fill_is_allocated(), - "We must have allocated the region we're returning"); - first = false; - } - return res; -} - -void G1CollectedHeap::remove_allocated_regions_from_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - { - HeapRegion* prev = NULL; - HeapRegion* cur = _unclean_region_list.hd(); - while (cur != NULL) { - HeapRegion* next = cur->next_from_unclean_list(); - if (cur->zero_fill_is_allocated()) { - // Remove from the list. - if (prev == NULL) { - (void)_unclean_region_list.pop(); - } else { - _unclean_region_list.delete_after(prev); - } - cur->set_on_unclean_list(false); - cur->set_next_on_unclean_list(NULL); - } else { - prev = cur; - } - cur = next; - } - assert(_unclean_region_list.sz() == unclean_region_list_length(), - "Inv"); +void G1CollectedHeap::wait_while_free_regions_coming() { + // Most of the time we won't have to wait, so let's do a quick test + // first before we take the lock. + if (!free_regions_coming()) { + return; + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "waiting for free regions"); } { - HeapRegion* prev = NULL; - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - HeapRegion* next = cur->next_from_free_list(); - if (cur->zero_fill_is_allocated()) { - // Remove from the list. - if (prev == NULL) { - _free_region_list = cur->next_from_free_list(); - } else { - prev->set_next_on_free_list(cur->next_from_free_list()); - } - cur->set_on_free_list(false); - cur->set_next_on_free_list(NULL); - _free_region_list_size--; - } else { - prev = cur; - } - cur = next; + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (free_regions_coming()) { + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); } - assert(_free_region_list_size == free_region_list_length(), "Inv"); - } -} - -bool G1CollectedHeap::verify_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - return verify_region_lists_locked(); -} - -bool G1CollectedHeap::verify_region_lists_locked() { - HeapRegion* unclean = _unclean_region_list.hd(); - while (unclean != NULL) { - guarantee(unclean->is_on_unclean_list(), "Well, it is!"); - guarantee(!unclean->is_on_free_list(), "Well, it shouldn't be!"); - guarantee(unclean->zero_fill_state() != HeapRegion::Allocated, - "Everything else is possible."); - unclean = unclean->next_from_unclean_list(); - } - guarantee(_unclean_region_list.sz() == unclean_region_list_length(), "Inv"); - - HeapRegion* free_r = _free_region_list; - while (free_r != NULL) { - assert(free_r->is_on_free_list(), "Well, it is!"); - assert(!free_r->is_on_unclean_list(), "Well, it shouldn't be!"); - switch (free_r->zero_fill_state()) { - case HeapRegion::NotZeroFilled: - case HeapRegion::ZeroFilling: - guarantee(false, "Should not be on free list."); - break; - default: - // Everything else is possible. - break; - } - free_r = free_r->next_from_free_list(); - } - guarantee(_free_region_list_size == free_region_list_length(), "Inv"); - // If we didn't do an assertion... - return true; -} - -size_t G1CollectedHeap::free_region_list_length() { - assert(ZF_mon->owned_by_self(), "precondition."); - size_t len = 0; - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - len++; - cur = cur->next_from_free_list(); - } - return len; -} - -size_t G1CollectedHeap::unclean_region_list_length() { - assert(ZF_mon->owned_by_self(), "precondition."); - return _unclean_region_list.length(); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "done waiting for free regions"); + } } size_t G1CollectedHeap::n_regions() { @@ -5454,55 +5312,6 @@ HeapRegion::GrainBytes; } -size_t G1CollectedHeap::free_regions() { - /* Possibly-expensive assert. - assert(_free_regions == count_free_regions(), - "_free_regions is off."); - */ - return _free_regions; -} - -bool G1CollectedHeap::should_zf() { - return _free_region_list_size < (size_t) G1ConcZFMaxRegions; -} - -class RegionCounter: public HeapRegionClosure { - size_t _n; -public: - RegionCounter() : _n(0) {} - bool doHeapRegion(HeapRegion* r) { - if (r->is_empty()) { - assert(!r->isHumongous(), "H regions should not be empty."); - _n++; - } - return false; - } - int res() { return (int) _n; } -}; - -size_t G1CollectedHeap::count_free_regions() { - RegionCounter rc; - heap_region_iterate(&rc); - size_t n = rc.res(); - if (_cur_alloc_region != NULL && _cur_alloc_region->is_empty()) - n--; - return n; -} - -size_t G1CollectedHeap::count_free_regions_list() { - size_t n = 0; - size_t o = 0; - ZF_mon->lock_without_safepoint_check(); - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - cur = cur->next_from_free_list(); - n++; - } - size_t m = unclean_region_list_length(); - ZF_mon->unlock(); - return n + m; -} - void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { assert(heap_lock_held_for_gc(), "the heap lock should already be held by or for this thread"); @@ -5574,28 +5383,19 @@ } } - // Done at the start of full GC. void G1CollectedHeap::tear_down_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - while (pop_unclean_region_list_locked() != NULL) ; - assert(_unclean_region_list.hd() == NULL && _unclean_region_list.sz() == 0, - "Postconditions of loop."); - while (pop_free_region_list_locked() != NULL) ; - assert(_free_region_list == NULL, "Postcondition of loop."); - if (_free_region_list_size != 0) { - gclog_or_tty->print_cr("Size is %d.", _free_region_list_size); - print_on(gclog_or_tty, true /* extended */); - } - assert(_free_region_list_size == 0, "Postconditions of loop."); + _free_list.remove_all(); } - class RegionResetter: public HeapRegionClosure { - G1CollectedHeap* _g1; - int _n; + G1CollectedHeap* _g1h; + FreeRegionList _local_free_list; + public: - RegionResetter() : _g1(G1CollectedHeap::heap()), _n(0) {} + RegionResetter() : _g1h(G1CollectedHeap::heap()), + _local_free_list("Local Free List for RegionResetter") { } + bool doHeapRegion(HeapRegion* r) { if (r->continuesHumongous()) return false; if (r->top() > r->bottom()) { @@ -5603,152 +5403,32 @@ Copy::fill_to_words(r->top(), pointer_delta(r->end(), r->top())); } - r->set_zero_fill_allocated(); } else { assert(r->is_empty(), "tautology"); - _n++; - switch (r->zero_fill_state()) { - case HeapRegion::NotZeroFilled: - case HeapRegion::ZeroFilling: - _g1->put_region_on_unclean_list_locked(r); - break; - case HeapRegion::Allocated: - r->set_zero_fill_complete(); - // no break; go on to put on free list. - case HeapRegion::ZeroFilled: - _g1->put_free_region_on_list_locked(r); - break; - } + _local_free_list.add_as_tail(r); } return false; } - int getFreeRegionCount() {return _n;} + void update_free_lists() { + _g1h->update_sets_after_freeing_regions(0, &_local_free_list, NULL, + false /* par */); + } }; // Done at the end of full GC. void G1CollectedHeap::rebuild_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); // This needs to go at the end of the full GC. RegionResetter rs; heap_region_iterate(&rs); - _free_regions = rs.getFreeRegionCount(); - // Tell the ZF thread it may have work to do. - if (should_zf()) ZF_mon->notify_all(); -} - -class UsedRegionsNeedZeroFillSetter: public HeapRegionClosure { - G1CollectedHeap* _g1; - int _n; -public: - UsedRegionsNeedZeroFillSetter() : _g1(G1CollectedHeap::heap()), _n(0) {} - bool doHeapRegion(HeapRegion* r) { - if (r->continuesHumongous()) return false; - if (r->top() > r->bottom()) { - // There are assertions in "set_zero_fill_needed()" below that - // require top() == bottom(), so this is technically illegal. - // We'll skirt the law here, by making that true temporarily. - DEBUG_ONLY(HeapWord* save_top = r->top(); - r->set_top(r->bottom())); - r->set_zero_fill_needed(); - DEBUG_ONLY(r->set_top(save_top)); - } - return false; - } -}; - -// Done at the start of full GC. -void G1CollectedHeap::set_used_regions_to_need_zero_fill() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - // This needs to go at the end of the full GC. - UsedRegionsNeedZeroFillSetter rs; - heap_region_iterate(&rs); + rs.update_free_lists(); } void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { _refine_cte_cl->set_concurrent(concurrent); } -#ifndef PRODUCT - -class PrintHeapRegionClosure: public HeapRegionClosure { -public: - bool doHeapRegion(HeapRegion *r) { - gclog_or_tty->print("Region: "PTR_FORMAT":", r); - if (r != NULL) { - if (r->is_on_free_list()) - gclog_or_tty->print("Free "); - if (r->is_young()) - gclog_or_tty->print("Young "); - if (r->isHumongous()) - gclog_or_tty->print("Is Humongous "); - r->print(); - } - return false; - } -}; - -class SortHeapRegionClosure : public HeapRegionClosure { - size_t young_regions,free_regions, unclean_regions; - size_t hum_regions, count; - size_t unaccounted, cur_unclean, cur_alloc; - size_t total_free; - HeapRegion* cur; -public: - SortHeapRegionClosure(HeapRegion *_cur) : cur(_cur), young_regions(0), - free_regions(0), unclean_regions(0), - hum_regions(0), - count(0), unaccounted(0), - cur_alloc(0), total_free(0) - {} - bool doHeapRegion(HeapRegion *r) { - count++; - if (r->is_on_free_list()) free_regions++; - else if (r->is_on_unclean_list()) unclean_regions++; - else if (r->isHumongous()) hum_regions++; - else if (r->is_young()) young_regions++; - else if (r == cur) cur_alloc++; - else unaccounted++; - return false; - } - void print() { - total_free = free_regions + unclean_regions; - gclog_or_tty->print("%d regions\n", count); - gclog_or_tty->print("%d free: free_list = %d unclean = %d\n", - total_free, free_regions, unclean_regions); - gclog_or_tty->print("%d humongous %d young\n", - hum_regions, young_regions); - gclog_or_tty->print("%d cur_alloc\n", cur_alloc); - gclog_or_tty->print("UHOH unaccounted = %d\n", unaccounted); - } -}; - -void G1CollectedHeap::print_region_counts() { - SortHeapRegionClosure sc(_cur_alloc_region); - PrintHeapRegionClosure cl; - heap_region_iterate(&cl); - heap_region_iterate(&sc); - sc.print(); - print_region_accounting_info(); -}; - -bool G1CollectedHeap::regions_accounted_for() { - // TODO: regions accounting for young/survivor/tenured - return true; -} - -bool G1CollectedHeap::print_region_accounting_info() { - gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", - free_regions(), - count_free_regions(), count_free_regions_list(), - _free_region_list_size, _unclean_region_list.sz()); - gclog_or_tty->print_cr("cur_alloc: %d.", - (_cur_alloc_region == NULL ? 0 : 1)); - gclog_or_tty->print_cr("H regions: %d.", _num_humongous_regions); - - // TODO: check regions accounting for young/survivor/tenured - return true; -} +#ifdef ASSERT bool G1CollectedHeap::is_in_closed_subset(const void* p) const { HeapRegion* hr = heap_region_containing(p); @@ -5758,8 +5438,84 @@ return hr->is_in(p); } } -#endif // !PRODUCT - -void G1CollectedHeap::g1_unimplemented() { - // Unimplemented(); +#endif // ASSERT + +class VerifyRegionListsClosure : public HeapRegionClosure { +private: + HumongousRegionSet* _humongous_set; + FreeRegionList* _free_list; + size_t _region_count; + +public: + VerifyRegionListsClosure(HumongousRegionSet* humongous_set, + FreeRegionList* free_list) : + _humongous_set(humongous_set), _free_list(free_list), + _region_count(0) { } + + size_t region_count() { return _region_count; } + + bool doHeapRegion(HeapRegion* hr) { + _region_count += 1; + + if (hr->continuesHumongous()) { + return false; + } + + if (hr->is_young()) { + // TODO + } else if (hr->startsHumongous()) { + _humongous_set->verify_next_region(hr); + } else if (hr->is_empty()) { + _free_list->verify_next_region(hr); + } + return false; + } +}; + +void G1CollectedHeap::verify_region_sets() { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + // First, check the explicit lists. + _free_list.verify(); + { + // Given that a concurrent operation might be adding regions to + // the secondary free list we have to take the lock before + // verifying it. + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _secondary_free_list.verify(); + } + _humongous_set.verify(); + + // If a concurrent region freeing operation is in progress it will + // be difficult to correctly attributed any free regions we come + // across to the correct free list given that they might belong to + // one of several (free_list, secondary_free_list, any local lists, + // etc.). So, if that's the case we will skip the rest of the + // verification operation. Alternatively, waiting for the concurrent + // operation to complete will have a non-trivial effect on the GC's + // operation (no concurrent operation will last longer than the + // interval between two calls to verification) and it might hide + // any issues that we would like to catch during testing. + if (free_regions_coming()) { + return; + } + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + // Make sure we append the secondary_free_list on the free_list so + // that all free regions we will come across can be safely + // attributed to the free_list. + append_secondary_free_list(); + } + + // Finally, make sure that the region accounting in the lists is + // consistent with what we see in the heap. + _humongous_set.verify_start(); + _free_list.verify_start(); + + VerifyRegionListsClosure cl(&_humongous_set, &_free_list); + heap_region_iterate(&cl); + + _humongous_set.verify_end(); + _free_list.verify_end(); }
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,7 +27,7 @@ #include "gc_implementation/g1/concurrentMark.hpp" #include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/g1/heapRegionSets.hpp" #include "gc_implementation/parNew/parGCAllocBuffer.hpp" #include "memory/barrierSet.hpp" #include "memory/memRegion.hpp" @@ -66,8 +66,7 @@ enum G1GCThreadGroups { G1CRGroup = 0, G1ZFGroup = 1, - G1CMGroup = 2, - G1CLGroup = 3 + G1CMGroup = 2 }; enum GCAllocPurpose { @@ -155,6 +154,7 @@ friend class RefineCardTableEntryClosure; friend class G1PrepareCompactClosure; friend class RegionSorter; + friend class RegionResetter; friend class CountRCClosure; friend class EvacPopObjClosure; friend class G1ParCleanupCTTask; @@ -178,17 +178,20 @@ // The maximum part of _g1_storage that has ever been committed. MemRegion _g1_max_committed; - // The number of regions that are completely free. - size_t _free_regions; + // The master free list. It will satisfy all new region allocations. + MasterFreeRegionList _free_list; + + // The secondary free list which contains regions that have been + // freed up during the cleanup process. This will be appended to the + // master free list when appropriate. + SecondaryFreeRegionList _secondary_free_list; + + // It keeps track of the humongous regions. + MasterHumongousRegionSet _humongous_set; // The number of regions we could create by expansion. size_t _expansion_regions; - // Return the number of free regions in the heap (by direct counting.) - size_t count_free_regions(); - // Return the number of free regions on the free and unclean lists. - size_t count_free_regions_list(); - // The block offset table for the G1 heap. G1BlockOffsetSharedArray* _bot_shared; @@ -196,9 +199,6 @@ // lists, before and after full GC. void tear_down_region_lists(); void rebuild_region_lists(); - // This sets all non-empty regions to need zero-fill (which they will if - // they are empty after full collection.) - void set_used_regions_to_need_zero_fill(); // The sequence of all heap regions in the heap. HeapRegionSeq* _hrs; @@ -231,7 +231,7 @@ // Determines PLAB size for a particular allocation purpose. static size_t desired_plab_sz(GCAllocPurpose purpose); - // When called by par thread, require par_alloc_during_gc_lock() to be held. + // When called by par thread, requires the FreeList_lock to be held. void push_gc_alloc_region(HeapRegion* hr); // This should only be called single-threaded. Undeclares all GC alloc @@ -294,10 +294,11 @@ // line number, file, etc. #define heap_locking_asserts_err_msg(__extra_message) \ - err_msg("%s : Heap_lock %slocked, %sat a safepoint", \ + err_msg("%s : Heap_lock locked: %s, at safepoint: %s, is VM thread: %s", \ (__extra_message), \ - (!Heap_lock->owned_by_self()) ? "NOT " : "", \ - (!SafepointSynchronize::is_at_safepoint()) ? "NOT " : "") + BOOL_TO_STR(Heap_lock->owned_by_self()), \ + BOOL_TO_STR(SafepointSynchronize::is_at_safepoint()), \ + BOOL_TO_STR(Thread::current()->is_VM_thread())) #define assert_heap_locked() \ do { \ @@ -305,10 +306,11 @@ heap_locking_asserts_err_msg("should be holding the Heap_lock")); \ } while (0) -#define assert_heap_locked_or_at_safepoint() \ +#define assert_heap_locked_or_at_safepoint(__should_be_vm_thread) \ do { \ assert(Heap_lock->owned_by_self() || \ - SafepointSynchronize::is_at_safepoint(), \ + (SafepointSynchronize::is_at_safepoint() && \ + ((__should_be_vm_thread) == Thread::current()->is_VM_thread())), \ heap_locking_asserts_err_msg("should be holding the Heap_lock or " \ "should be at a safepoint")); \ } while (0) @@ -335,9 +337,10 @@ "should not be at a safepoint")); \ } while (0) -#define assert_at_safepoint() \ +#define assert_at_safepoint(__should_be_vm_thread) \ do { \ - assert(SafepointSynchronize::is_at_safepoint(), \ + assert(SafepointSynchronize::is_at_safepoint() && \ + ((__should_be_vm_thread) == Thread::current()->is_VM_thread()), \ heap_locking_asserts_err_msg("should be at a safepoint")); \ } while (0) @@ -362,31 +365,41 @@ // The current policy object for the collector. G1CollectorPolicy* _g1_policy; - // Parallel allocation lock to protect the current allocation region. - Mutex _par_alloc_during_gc_lock; - Mutex* par_alloc_during_gc_lock() { return &_par_alloc_during_gc_lock; } + // This is the second level of trying to allocate a new region. If + // new_region_work didn't find a region in the free_list, this call + // will check whether there's anything available in the + // secondary_free_list and/or wait for more regions to appear in that + // list, if _free_regions_coming is set. + HeapRegion* new_region_try_secondary_free_list(size_t word_size); - // If possible/desirable, allocate a new HeapRegion for normal object - // allocation sufficient for an allocation of the given "word_size". - // If "do_expand" is true, will attempt to expand the heap if necessary - // to to satisfy the request. If "zero_filled" is true, requires a - // zero-filled region. - // (Returning NULL will trigger a GC.) - virtual HeapRegion* newAllocRegion_work(size_t word_size, - bool do_expand, - bool zero_filled); + // It will try to allocate a single non-humongous HeapRegion + // sufficient for an allocation of the given word_size. If + // do_expand is true, it will attempt to expand the heap if + // necessary to satisfy the allocation request. Note that word_size + // is only used to make sure that we expand sufficiently but, given + // that the allocation request is assumed not to be humongous, + // having word_size is not strictly necessary (expanding by a single + // region will always be sufficient). But let's keep that parameter + // in case we need it in the future. + HeapRegion* new_region_work(size_t word_size, bool do_expand); - virtual HeapRegion* newAllocRegion(size_t word_size, - bool zero_filled = true) { - return newAllocRegion_work(word_size, false, zero_filled); + // It will try to allocate a new region to be used for allocation by + // mutator threads. It will not try to expand the heap if not region + // is available. + HeapRegion* new_alloc_region(size_t word_size) { + return new_region_work(word_size, false /* do_expand */); } - virtual HeapRegion* newAllocRegionWithExpansion(int purpose, - size_t word_size, - bool zero_filled = true); + + // It will try to allocate a new region to be used for allocation by + // a GC thread. It will try to expand the heap if no region is + // available. + HeapRegion* new_gc_alloc_region(int purpose, size_t word_size); + + int humongous_obj_allocate_find_first(size_t num_regions, size_t word_size); // Attempt to allocate an object of the given (very large) "word_size". // Returns "NULL" on failure. - virtual HeapWord* humongous_obj_allocate(size_t word_size); + HeapWord* humongous_obj_allocate(size_t word_size); // The following two methods, allocate_new_tlab() and // mem_allocate(), are the two main entry points from the runtime @@ -430,7 +443,8 @@ bool* gc_overhead_limit_was_exceeded); // The following methods, allocate_from_cur_allocation_region(), - // attempt_allocation(), replace_cur_alloc_region_and_allocate(), + // attempt_allocation(), attempt_allocation_locked(), + // replace_cur_alloc_region_and_allocate(), // attempt_allocation_slow(), and attempt_allocation_humongous() // have very awkward pre- and post-conditions with respect to // locking: @@ -481,20 +495,30 @@ // successfully manage to allocate it, or NULL. // It tries to satisfy an allocation request out of the current - // allocating region, which is passed as a parameter. It assumes - // that the caller has checked that the current allocating region is - // not NULL. Given that the caller has to check the current - // allocating region for at least NULL, it might as well pass it as - // the first parameter so that the method doesn't have to read it - // from the _cur_alloc_region field again. + // alloc region, which is passed as a parameter. It assumes that the + // caller has checked that the current alloc region is not NULL. + // Given that the caller has to check the current alloc region for + // at least NULL, it might as well pass it as the first parameter so + // that the method doesn't have to read it from the + // _cur_alloc_region field again. It is called from both + // attempt_allocation() and attempt_allocation_locked() and the + // with_heap_lock parameter indicates whether the caller was holding + // the heap lock when it called it or not. inline HeapWord* allocate_from_cur_alloc_region(HeapRegion* cur_alloc_region, - size_t word_size); + size_t word_size, + bool with_heap_lock); - // It attempts to allocate out of the current alloc region. If that - // fails, it retires the current alloc region (if there is one), - // tries to get a new one and retries the allocation. + // First-level of allocation slow path: it attempts to allocate out + // of the current alloc region in a lock-free manner using a CAS. If + // that fails it takes the Heap_lock and calls + // attempt_allocation_locked() for the second-level slow path. inline HeapWord* attempt_allocation(size_t word_size); + // Second-level of allocation slow path: while holding the Heap_lock + // it tries to allocate out of the current alloc region and, if that + // fails, tries to allocate out of a new current alloc region. + inline HeapWord* attempt_allocation_locked(size_t word_size); + // It assumes that the current alloc region has been retired and // tries to allocate a new one. If it's successful, it performs the // allocation out of the new current alloc region and updates @@ -506,11 +530,11 @@ bool do_dirtying, bool can_expand); - // The slow path when we are unable to allocate a new current alloc - // region to satisfy an allocation request (i.e., when - // attempt_allocation() fails). It will try to do an evacuation - // pause, which might stall due to the GC locker, and retry the - // allocation attempt when appropriate. + // Third-level of allocation slow path: when we are unable to + // allocate a new current alloc region to satisfy an allocation + // request (i.e., when attempt_allocation_locked() fails). It will + // try to do an evacuation pause, which might stall due to the GC + // locker, and retry the allocation attempt when appropriate. HeapWord* attempt_allocation_slow(size_t word_size); // The method that tries to satisfy a humongous allocation @@ -749,20 +773,29 @@ // Invoke "save_marks" on all heap regions. void save_marks(); - // Free a heap region. - void free_region(HeapRegion* hr); - // A component of "free_region", exposed for 'batching'. - // All the params after "hr" are out params: the used bytes of the freed - // region(s), the number of H regions cleared, the number of regions - // freed, and pointers to the head and tail of a list of freed contig - // regions, linked throught the "next_on_unclean_list" field. - void free_region_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h, - size_t& freed_regions, - UncleanRegionList* list, - bool par = false); + // It frees a non-humongous region by initializing its contents and + // adding it to the free list that's passed as a parameter (this is + // usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + void free_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + bool par); + // It frees a humongous region by collapsing it into individual + // regions and calling free_region() for each of them. The freed + // regions will be added to the free list that's passed as a parameter + // (this is usually a local list which will be appended to the + // master free list later). The used bytes of freed regions are + // accumulated in pre_used. If par is true, the region's RSet will + // not be freed up. The assumption is that this will be done later. + void free_humongous_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); // The concurrent marker (and the thread it runs in.) ConcurrentMark* _cm; @@ -772,9 +805,6 @@ // The concurrent refiner. ConcurrentG1Refine* _cg1r; - // The concurrent zero-fill thread. - ConcurrentZFThread* _czft; - // The parallel task queues RefToScanQueueSet *_task_queues; @@ -826,7 +856,6 @@ void finalize_for_evac_failure(); // An attempt to evacuate "obj" has failed; take necessary steps. - void handle_evacuation_failure(oop obj); oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj); void handle_evacuation_failure_common(oop obj, markOop m); @@ -867,9 +896,7 @@ SubTasksDone* _process_strong_tasks; - // List of regions which require zero filling. - UncleanRegionList _unclean_region_list; - bool _unclean_regions_coming; + volatile bool _free_regions_coming; public: @@ -992,71 +1019,64 @@ size_t max_regions(); // The number of regions that are completely free. - size_t free_regions(); + size_t free_regions() { + return _free_list.length(); + } // The number of regions that are not completely free. size_t used_regions() { return n_regions() - free_regions(); } - // True iff the ZF thread should run. - bool should_zf(); - // The number of regions available for "regular" expansion. size_t expansion_regions() { return _expansion_regions; } -#ifndef PRODUCT - bool regions_accounted_for(); - bool print_region_accounting_info(); - void print_region_counts(); -#endif - - HeapRegion* alloc_region_from_unclean_list(bool zero_filled); - HeapRegion* alloc_region_from_unclean_list_locked(bool zero_filled); - - void put_region_on_unclean_list(HeapRegion* r); - void put_region_on_unclean_list_locked(HeapRegion* r); + // verify_region_sets() performs verification over the region + // lists. It will be compiled in the product code to be used when + // necessary (i.e., during heap verification). + void verify_region_sets(); - void prepend_region_list_on_unclean_list(UncleanRegionList* list); - void prepend_region_list_on_unclean_list_locked(UncleanRegionList* list); + // verify_region_sets_optional() is planted in the code for + // list verification in non-product builds (and it can be enabled in + // product builds by definning HEAP_REGION_SET_FORCE_VERIFY to be 1). +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { + verify_region_sets(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY - void set_unclean_regions_coming(bool b); - void set_unclean_regions_coming_locked(bool b); - // Wait for cleanup to be complete. - void wait_for_cleanup_complete(); - // Like above, but assumes that the calling thread owns the Heap_lock. - void wait_for_cleanup_complete_locked(); - - // Return the head of the unclean list. - HeapRegion* peek_unclean_region_list_locked(); - // Remove and return the head of the unclean list. - HeapRegion* pop_unclean_region_list_locked(); +#ifdef ASSERT + bool is_on_free_list(HeapRegion* hr) { + return hr->containing_set() == &_free_list; + } - // List of regions which are zero filled and ready for allocation. - HeapRegion* _free_region_list; - // Number of elements on the free list. - size_t _free_region_list_size; + bool is_on_humongous_set(HeapRegion* hr) { + return hr->containing_set() == &_humongous_set; +} +#endif // ASSERT - // If the head of the unclean list is ZeroFilled, move it to the free - // list. - bool move_cleaned_region_to_free_list_locked(); - bool move_cleaned_region_to_free_list(); + // Wrapper for the region list operations that can be called from + // methods outside this class. - void put_free_region_on_list_locked(HeapRegion* r); - void put_free_region_on_list(HeapRegion* r); + void secondary_free_list_add_as_tail(FreeRegionList* list) { + _secondary_free_list.add_as_tail(list); + } - // Remove and return the head element of the free list. - HeapRegion* pop_free_region_list_locked(); + void append_secondary_free_list() { + _free_list.add_as_tail(&_secondary_free_list); + } - // If "zero_filled" is true, we first try the free list, then we try the - // unclean list, zero-filling the result. If "zero_filled" is false, we - // first try the unclean list, then the zero-filled list. - HeapRegion* alloc_free_region_from_lists(bool zero_filled); + void append_secondary_free_list_if_not_empty() { + if (!_secondary_free_list.is_empty()) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + append_secondary_free_list(); + } + } - // Verify the integrity of the region lists. - void remove_allocated_regions_from_lists(); - bool verify_region_lists(); - bool verify_region_lists_locked(); - size_t unclean_region_list_length(); - size_t free_region_list_length(); + void set_free_regions_coming(); + void reset_free_regions_coming(); + bool free_regions_coming() { return _free_regions_coming; } + void wait_while_free_regions_coming(); // Perform a collection of the heap; intended for use in implementing // "System.gc". This probably implies as full a collection as the @@ -1075,23 +1095,24 @@ // True iff a evacuation has failed in the most-recent collection. bool evacuation_failed() { return _evacuation_failed; } - // Free a region if it is totally full of garbage. Returns the number of - // bytes freed (0 ==> didn't free it). - size_t free_region_if_totally_empty(HeapRegion *hr); - void free_region_if_totally_empty_work(HeapRegion *hr, - size_t& pre_used, - size_t& cleared_h_regions, - size_t& freed_regions, - UncleanRegionList* list, - bool par = false); + // It will free a region if it has allocated objects in it that are + // all dead. It calls either free_region() or + // free_humongous_region() depending on the type of the region that + // is passed to it. + void free_region_if_totally_empty(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); - // If we've done free region work that yields the given changes, update - // the relevant global variables. - void finish_free_region_work(size_t pre_used, - size_t cleared_h_regions, - size_t freed_regions, - UncleanRegionList* list); - + // It appends the free list to the master free list and updates the + // master humongous list according to the contents of the proxy + // list. It also adjusts the total used bytes according to pre_used + // (if par is true, it will do so by taking the ParGCRareEvent_lock). + void update_sets_after_freeing_regions(size_t pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); // Returns "TRUE" iff "p" points into the allocated area of the heap. virtual bool is_in(const void* p) const; @@ -1304,8 +1325,6 @@ return true; } - virtual bool allocs_are_zero_filled(); - // The boundary between a "large" and "small" array of primitives, in // words. virtual size_t large_typearray_limit(); @@ -1536,13 +1555,6 @@ protected: size_t _max_heap_capacity; - -public: - // Temporary: call to mark things unimplemented for the G1 heap (e.g., - // MemoryService). In productization, we can make this assert false - // to catch such places (as well as searching for calls to this...) - static void g1_unimplemented(); - }; #define use_local_bitmaps 1
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,7 @@ #include "gc_implementation/g1/concurrentMark.hpp" #include "gc_implementation/g1/g1CollectedHeap.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/heapRegionSeq.hpp" +#include "gc_implementation/g1/heapRegionSeq.inline.hpp" #include "utilities/taskqueue.hpp" // Inline functions for G1CollectedHeap @@ -63,10 +63,12 @@ // assumptions of this method (and other related ones). inline HeapWord* G1CollectedHeap::allocate_from_cur_alloc_region(HeapRegion* cur_alloc_region, - size_t word_size) { - assert_heap_locked_and_not_at_safepoint(); + size_t word_size, + bool with_heap_lock) { + assert_not_at_safepoint(); + assert(with_heap_lock == Heap_lock->owned_by_self(), + "with_heap_lock and Heap_lock->owned_by_self() should be a tautology"); assert(cur_alloc_region != NULL, "pre-condition of the method"); - assert(cur_alloc_region == _cur_alloc_region, "pre-condition of the method"); assert(cur_alloc_region->is_young(), "we only support young current alloc regions"); assert(!isHumongous(word_size), "allocate_from_cur_alloc_region() " @@ -76,20 +78,24 @@ assert(!cur_alloc_region->is_empty(), err_msg("region ["PTR_FORMAT","PTR_FORMAT"] should not be empty", cur_alloc_region->bottom(), cur_alloc_region->end())); - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = cur_alloc_region->allocate(word_size); + HeapWord* result = cur_alloc_region->par_allocate_no_bot_updates(word_size); if (result != NULL) { assert(is_in(result), "result should be in the heap"); - Heap_lock->unlock(); + if (with_heap_lock) { + Heap_lock->unlock(); + } + assert_heap_not_locked(); // Do the dirtying after we release the Heap_lock. dirty_young_block(result, word_size); return result; } - assert_heap_locked(); + if (with_heap_lock) { + assert_heap_locked(); + } else { + assert_heap_not_locked(); + } return NULL; } @@ -97,26 +103,75 @@ // assumptions of this method (and other related ones). inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size) { - assert_heap_locked_and_not_at_safepoint(); + assert_heap_not_locked_and_not_at_safepoint(); assert(!isHumongous(word_size), "attempt_allocation() should not be called " "for humongous allocation requests"); HeapRegion* cur_alloc_region = _cur_alloc_region; if (cur_alloc_region != NULL) { HeapWord* result = allocate_from_cur_alloc_region(cur_alloc_region, - word_size); + word_size, + false /* with_heap_lock */); + assert_heap_not_locked(); + if (result != NULL) { + return result; + } + } + + // Our attempt to allocate lock-free failed as the current + // allocation region is either NULL or full. So, we'll now take the + // Heap_lock and retry. + Heap_lock->lock(); + + HeapWord* result = attempt_allocation_locked(word_size); + if (result != NULL) { + assert_heap_not_locked(); + return result; + } + + assert_heap_locked(); + return NULL; +} + +inline void +G1CollectedHeap::retire_cur_alloc_region_common(HeapRegion* cur_alloc_region) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + assert(cur_alloc_region != NULL && cur_alloc_region == _cur_alloc_region, + "pre-condition of the call"); + assert(cur_alloc_region->is_young(), + "we only support young current alloc regions"); + + // The region is guaranteed to be young + g1_policy()->add_region_to_incremental_cset_lhs(cur_alloc_region); + _summary_bytes_used += cur_alloc_region->used(); + _cur_alloc_region = NULL; +} + +inline HeapWord* +G1CollectedHeap::attempt_allocation_locked(size_t word_size) { + assert_heap_locked_and_not_at_safepoint(); + assert(!isHumongous(word_size), "attempt_allocation_locked() " + "should not be called for humongous allocation requests"); + + // First, reread the current alloc region and retry the allocation + // in case somebody replaced it while we were waiting to get the + // Heap_lock. + HeapRegion* cur_alloc_region = _cur_alloc_region; + if (cur_alloc_region != NULL) { + HeapWord* result = allocate_from_cur_alloc_region( + cur_alloc_region, word_size, + true /* with_heap_lock */); if (result != NULL) { assert_heap_not_locked(); return result; } - assert_heap_locked(); - - // Since we couldn't successfully allocate into it, retire the - // current alloc region. + // We failed to allocate out of the current alloc region, so let's + // retire it before getting a new one. retire_cur_alloc_region(cur_alloc_region); } + assert_heap_locked(); // Try to get a new region and allocate out of it HeapWord* result = replace_cur_alloc_region_and_allocate(word_size, false, /* at_safepoint */ @@ -131,20 +186,6 @@ return NULL; } -inline void -G1CollectedHeap::retire_cur_alloc_region_common(HeapRegion* cur_alloc_region) { - assert_heap_locked_or_at_safepoint(); - assert(cur_alloc_region != NULL && cur_alloc_region == _cur_alloc_region, - "pre-condition of the call"); - assert(cur_alloc_region->is_young(), - "we only support young current alloc regions"); - - // The region is guaranteed to be young - g1_policy()->add_region_to_incremental_cset_lhs(cur_alloc_region); - _summary_bytes_used += cur_alloc_region->used(); - _cur_alloc_region = NULL; -} - // It dirties the cards that cover the block so that so that the post // write barrier never queues anything when updating objects on this // block. It is assumed (and in fact we assert) that the block
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2875,8 +2875,6 @@ // Adjust for expansion and slop. max_live_bytes = max_live_bytes + expansion_bytes; - assert(_g1->regions_accounted_for(), "Region leakage!"); - HeapRegion* hr; if (in_young_gc_mode()) { double young_start_time_sec = os::elapsedTime();
--- a/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -113,6 +113,7 @@ Threads::gc_epilogue(); CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); // refs processing: clean slate GenMarkSweep::_ref_processor = NULL; @@ -180,26 +181,46 @@ } class G1PrepareCompactClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; ModRefBarrierSet* _mrbs; CompactPoint _cp; + size_t _pre_used; + FreeRegionList _free_list; + HumongousRegionSet _humongous_proxy_set; void free_humongous_region(HeapRegion* hr) { - HeapWord* bot = hr->bottom(); HeapWord* end = hr->end(); assert(hr->startsHumongous(), "Only the start of a humongous region should be freed."); - G1CollectedHeap::heap()->free_region(hr); + _g1h->free_humongous_region(hr, &_pre_used, &_free_list, + &_humongous_proxy_set, false /* par */); + // Do we also need to do this for the continues humongous regions + // we just collapsed? hr->prepare_for_compaction(&_cp); // Also clear the part of the card table that will be unused after // compaction. - _mrbs->clear(MemRegion(hr->compaction_top(), hr->end())); + _mrbs->clear(MemRegion(hr->compaction_top(), end)); } public: - G1PrepareCompactClosure(CompactibleSpace* cs) : + G1PrepareCompactClosure(CompactibleSpace* cs) + : _g1h(G1CollectedHeap::heap()), + _mrbs(G1CollectedHeap::heap()->mr_bs()), _cp(NULL, cs, cs->initialize_threshold()), - _mrbs(G1CollectedHeap::heap()->mr_bs()) - {} + _pre_used(0), + _free_list("Local Free List for G1MarkSweep"), + _humongous_proxy_set("G1MarkSweep Humongous Proxy Set") { } + + void update_sets() { + // We'll recalculate total used bytes and recreate the free list + // at the end of the GC, so no point in updating those values here. + _g1h->update_sets_after_freeing_regions(0, /* pre_used */ + NULL, /* free_list */ + &_humongous_proxy_set, + false /* par */); + _free_list.remove_all(); + } + bool doHeapRegion(HeapRegion* hr) { if (hr->isHumongous()) { if (hr->startsHumongous()) { @@ -265,6 +286,7 @@ G1PrepareCompactClosure blk(sp); g1h->heap_region_iterate(&blk); + blk.update_sets(); CompactPoint perm_cp(pg, NULL, NULL); pg->prepare_for_compaction(&perm_cp);
--- a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,21 +75,12 @@ "(0 means do not periodically generate this info); " \ "it also requires -XX:+G1SummarizeRSetStats") \ \ - diagnostic(bool, G1SummarizeZFStats, false, \ - "Summarize zero-filling info") \ - \ diagnostic(bool, G1TraceConcRefinement, false, \ "Trace G1 concurrent refinement") \ \ product(intx, G1MarkRegionStackSize, 1024 * 1024, \ "Size of the region stack for concurrent marking.") \ \ - develop(bool, G1ConcZeroFill, true, \ - "If true, run concurrent zero-filling thread") \ - \ - develop(intx, G1ConcZFMaxRegions, 1, \ - "Stop zero-filling when # of zf'd regions reaches") \ - \ develop(bool, G1SATBBarrierPrintNullPreVals, false, \ "If true, count frac of ptr writes with null pre-vals.") \ \ @@ -99,6 +90,13 @@ develop(intx, G1SATBProcessCompletedThreshold, 20, \ "Number of completed buffers that triggers log processing.") \ \ + product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60, \ + "Before enqueueing them, each mutator thread tries to do some " \ + "filtering on the SATB buffers it generates. If post-filtering " \ + "the percentage of retained entries is over this threshold " \ + "the buffer will be enqueued for processing. A value of 0 " \ + "specifies that mutator threads should not do such filtering.") \ + \ develop(intx, G1ExtraRegionSurvRate, 33, \ "If the young survival rate is S, and there's room left in " \ "to-space, we will allow regions whose survival rate is up to " \ @@ -282,7 +280,20 @@ "Size of a work unit of cards claimed by a worker thread" \ "during RSet scanning.") \ \ - develop(bool, ReduceInitialCardMarksForG1, false, \ + develop(uintx, G1SecondaryFreeListAppendLength, 5, \ + "The number of regions we will add to the secondary free list " \ + "at every append operation") \ + \ + develop(bool, G1ConcRegionFreeingVerbose, false, \ + "Enables verboseness during concurrent region freeing") \ + \ + develop(bool, G1StressConcRegionFreeing, false, \ + "It stresses the concurrent region freeing operation") \ + \ + develop(uintx, G1StressConcRegionFreeingDelayMillis, 0, \ + "Artificial delay during concurrent region freeing") \ + \ + develop(bool, ReduceInitialCardMarksForG1, false, \ "When ReduceInitialCardMarks is true, this flag setting " \ " controls whether G1 allows the RICM optimization")
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,6 @@ */ #include "precompiled.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1OopClosures.inline.hpp" @@ -348,22 +347,20 @@ } void HeapRegion::hr_clear(bool par, bool clear_space) { - _humongous_type = NotHumongous; - _humongous_start_region = NULL; + assert(_humongous_type == NotHumongous, + "we should have already filtered out humongous regions"); + assert(_humongous_start_region == NULL, + "we should have already filtered out humongous regions"); + assert(_end == _orig_end, + "we should have already filtered out humongous regions"); + _in_collection_set = false; _is_gc_alloc_region = false; - // Age stuff (if parallel, this will be done separately, since it needs - // to be sequential). - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - set_young_index_in_cset(-1); uninstall_surv_rate_group(); set_young_type(NotYoung); - // In case it had been the start of a humongous sequence, reset its end. - set_end(_orig_end); - if (!par) { // If this is parallel, this will be done later. HeapRegionRemSet* hrrs = rem_set(); @@ -386,26 +383,49 @@ } // </PREDICTION> -void HeapRegion::set_startsHumongous(HeapWord* new_end) { +void HeapRegion::set_startsHumongous(HeapWord* new_top, HeapWord* new_end) { + assert(!isHumongous(), "sanity / pre-condition"); assert(end() == _orig_end, "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); + assert(bottom() <= new_top && new_top <= new_end, "pre-condition"); _humongous_type = StartsHumongous; _humongous_start_region = this; set_end(new_end); - _offsets.set_for_starts_humongous(new_end); + _offsets.set_for_starts_humongous(new_top); } -void HeapRegion::set_continuesHumongous(HeapRegion* start) { +void HeapRegion::set_continuesHumongous(HeapRegion* first_hr) { + assert(!isHumongous(), "sanity / pre-condition"); assert(end() == _orig_end, "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); - assert(start->startsHumongous(), "pre-condition"); + assert(first_hr->startsHumongous(), "pre-condition"); _humongous_type = ContinuesHumongous; - _humongous_start_region = start; + _humongous_start_region = first_hr; +} + +void HeapRegion::set_notHumongous() { + assert(isHumongous(), "pre-condition"); + + if (startsHumongous()) { + assert(top() <= end(), "pre-condition"); + set_end(_orig_end); + if (top() > end()) { + // at least one "continues humongous" region after it + set_top(end()); + } + } else { + // continues humongous + assert(end() == _orig_end, "sanity"); + } + + assert(capacity() == (size_t) HeapRegion::GrainBytes, "pre-condition"); + _humongous_type = NotHumongous; + _humongous_start_region = NULL; } bool HeapRegion::claimHeapRegion(jint claimValue) { @@ -442,15 +462,6 @@ return low; } -void HeapRegion::set_next_on_unclean_list(HeapRegion* r) { - assert(r == NULL || r->is_on_unclean_list(), "Malformed unclean list."); - _next_in_special_set = r; -} - -void HeapRegion::set_on_unclean_list(bool b) { - _is_on_unclean_list = b; -} - void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) { G1OffsetTableContigSpace::initialize(mr, false, mangle_space); hr_clear(false/*par*/, clear_space); @@ -468,15 +479,16 @@ _hrs_index(-1), _humongous_type(NotHumongous), _humongous_start_region(NULL), _in_collection_set(false), _is_gc_alloc_region(false), - _is_on_free_list(false), _is_on_unclean_list(false), _next_in_special_set(NULL), _orig_end(NULL), _claimed(InitialClaimValue), _evacuation_failed(false), _prev_marked_bytes(0), _next_marked_bytes(0), _sort_index(-1), _young_type(NotYoung), _next_young_region(NULL), - _next_dirty_cards_region(NULL), - _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), - _rem_set(NULL), _zfs(NotZeroFilled), - _recorded_rs_length(0), _predicted_elapsed_time_ms(0), + _next_dirty_cards_region(NULL), _next(NULL), _pending_removal(false), +#ifdef ASSERT + _containing_set(NULL), +#endif // ASSERT + _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), + _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), _predicted_bytes_to_copy(0) { _orig_end = mr.end(); @@ -551,86 +563,6 @@ oops_in_mr_iterate(MemRegion(bottom(), saved_mark_word()), cl); } -#ifdef DEBUG -HeapWord* HeapRegion::allocate(size_t size) { - jint state = zero_fill_state(); - assert(!G1CollectedHeap::heap()->allocs_are_zero_filled() || - zero_fill_is_allocated(), - "When ZF is on, only alloc in ZF'd regions"); - return G1OffsetTableContigSpace::allocate(size); -} -#endif - -void HeapRegion::set_zero_fill_state_work(ZeroFillState zfs) { - assert(ZF_mon->owned_by_self() || - Universe::heap()->is_gc_active(), - "Must hold the lock or be a full GC to modify."); -#ifdef ASSERT - if (top() != bottom() && zfs != Allocated) { - ResourceMark rm; - stringStream region_str; - print_on(®ion_str); - assert(top() == bottom() || zfs == Allocated, - err_msg("Region must be empty, or we must be setting it to allocated. " - "_zfs=%d, zfs=%d, region: %s", _zfs, zfs, region_str.as_string())); - } -#endif - _zfs = zfs; -} - -void HeapRegion::set_zero_fill_complete() { - set_zero_fill_state_work(ZeroFilled); - if (ZF_mon->owned_by_self()) { - ZF_mon->notify_all(); - } -} - - -void HeapRegion::ensure_zero_filled() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - ensure_zero_filled_locked(); -} - -void HeapRegion::ensure_zero_filled_locked() { - assert(ZF_mon->owned_by_self(), "Precondition"); - bool should_ignore_zf = SafepointSynchronize::is_at_safepoint(); - assert(should_ignore_zf || Heap_lock->is_locked(), - "Either we're in a GC or we're allocating a region."); - switch (zero_fill_state()) { - case HeapRegion::NotZeroFilled: - set_zero_fill_in_progress(Thread::current()); - { - ZF_mon->unlock(); - Copy::fill_to_words(bottom(), capacity()/HeapWordSize); - ZF_mon->lock_without_safepoint_check(); - } - // A trap. - guarantee(zero_fill_state() == HeapRegion::ZeroFilling - && zero_filler() == Thread::current(), - "AHA! Tell Dave D if you see this..."); - set_zero_fill_complete(); - // gclog_or_tty->print_cr("Did sync ZF."); - ConcurrentZFThread::note_sync_zfs(); - break; - case HeapRegion::ZeroFilling: - if (should_ignore_zf) { - // We can "break" the lock and take over the work. - Copy::fill_to_words(bottom(), capacity()/HeapWordSize); - set_zero_fill_complete(); - ConcurrentZFThread::note_sync_zfs(); - break; - } else { - ConcurrentZFThread::wait_for_ZF_completed(this); - } - case HeapRegion::ZeroFilled: - // Nothing to do. - break; - case HeapRegion::Allocated: - guarantee(false, "Should not call on allocated regions."); - } - assert(zero_fill_state() == HeapRegion::ZeroFilled, "Post"); -} - HeapWord* HeapRegion::object_iterate_mem_careful(MemRegion mr, ObjectClosure* cl) { @@ -782,9 +714,6 @@ verify(allow_dirty, /* use_prev_marking */ true, /* failures */ &dummy); } -#define OBJ_SAMPLE_INTERVAL 0 -#define BLOCK_SAMPLE_INTERVAL 100 - // This really ought to be commoned up into OffsetTableContigSpace somehow. // We would need a mechanism to make that code skip dead objects. @@ -795,83 +724,125 @@ *failures = false; HeapWord* p = bottom(); HeapWord* prev_p = NULL; - int objs = 0; - int blocks = 0; VerifyLiveClosure vl_cl(g1, use_prev_marking); bool is_humongous = isHumongous(); + bool do_bot_verify = !is_young(); size_t object_num = 0; while (p < top()) { - size_t size = oop(p)->size(); - if (is_humongous != g1->isHumongous(size)) { + oop obj = oop(p); + size_t obj_size = obj->size(); + object_num += 1; + + if (is_humongous != g1->isHumongous(obj_size)) { gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" SIZE_FORMAT" words) in a %shumongous region", - p, g1->isHumongous(size) ? "" : "non-", - size, is_humongous ? "" : "non-"); + p, g1->isHumongous(obj_size) ? "" : "non-", + obj_size, is_humongous ? "" : "non-"); *failures = true; + return; + } + + // If it returns false, verify_for_object() will output the + // appropriate messasge. + if (do_bot_verify && !_offsets.verify_for_object(p, obj_size)) { + *failures = true; + return; } - object_num += 1; - if (blocks == BLOCK_SAMPLE_INTERVAL) { - HeapWord* res = block_start_const(p + (size/2)); - if (p != res) { - gclog_or_tty->print_cr("offset computation 1 for "PTR_FORMAT" and " - SIZE_FORMAT" returned "PTR_FORMAT, - p, size, res); + + if (!g1->is_obj_dead_cond(obj, this, use_prev_marking)) { + if (obj->is_oop()) { + klassOop klass = obj->klass(); + if (!klass->is_perm()) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not in perm", klass, obj); + *failures = true; + return; + } else if (!klass->is_klass()) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not a klass", klass, obj); + *failures = true; + return; + } else { + vl_cl.set_containing_obj(obj); + obj->oop_iterate(&vl_cl); + if (vl_cl.failures()) { + *failures = true; + } + if (G1MaxVerifyFailures >= 0 && + vl_cl.n_failures() >= G1MaxVerifyFailures) { + return; + } + } + } else { + gclog_or_tty->print_cr(PTR_FORMAT" no an oop", obj); *failures = true; return; } - blocks = 0; - } else { - blocks++; - } - if (objs == OBJ_SAMPLE_INTERVAL) { - oop obj = oop(p); - if (!g1->is_obj_dead_cond(obj, this, use_prev_marking)) { - if (obj->is_oop()) { - klassOop klass = obj->klass(); - if (!klass->is_perm()) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not in perm", klass, obj); - *failures = true; - return; - } else if (!klass->is_klass()) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not a klass", klass, obj); - *failures = true; - return; - } else { - vl_cl.set_containing_obj(obj); - obj->oop_iterate(&vl_cl); - if (vl_cl.failures()) { - *failures = true; - } - if (G1MaxVerifyFailures >= 0 && - vl_cl.n_failures() >= G1MaxVerifyFailures) { - return; - } - } - } else { - gclog_or_tty->print_cr(PTR_FORMAT" no an oop", obj); - *failures = true; - return; - } - } - objs = 0; - } else { - objs++; } prev_p = p; - p += size; + p += obj_size; + } + + if (p != top()) { + gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " + "does not match top "PTR_FORMAT, p, top()); + *failures = true; + return; } - HeapWord* rend = end(); - HeapWord* rtop = top(); - if (rtop < rend) { - HeapWord* res = block_start_const(rtop + (rend - rtop) / 2); - if (res != rtop) { - gclog_or_tty->print_cr("offset computation 2 for "PTR_FORMAT" and " - PTR_FORMAT" returned "PTR_FORMAT, - rtop, rend, res); + + HeapWord* the_end = end(); + assert(p == top(), "it should still hold"); + // Do some extra BOT consistency checking for addresses in the + // range [top, end). BOT look-ups in this range should yield + // top. No point in doing that if top == end (there's nothing there). + if (p < the_end) { + // Look up top + HeapWord* addr_1 = p; + HeapWord* b_start_1 = _offsets.block_start_const(addr_1); + if (b_start_1 != p) { + gclog_or_tty->print_cr("BOT look up for top: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_1, b_start_1, p); + *failures = true; + return; + } + + // Look up top + 1 + HeapWord* addr_2 = p + 1; + if (addr_2 < the_end) { + HeapWord* b_start_2 = _offsets.block_start_const(addr_2); + if (b_start_2 != p) { + gclog_or_tty->print_cr("BOT look up for top + 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_2, b_start_2, p); *failures = true; return; + } + } + + // Look up an address between top and end + size_t diff = pointer_delta(the_end, p) / 2; + HeapWord* addr_3 = p + diff; + if (addr_3 < the_end) { + HeapWord* b_start_3 = _offsets.block_start_const(addr_3); + if (b_start_3 != p) { + gclog_or_tty->print_cr("BOT look up for top + diff: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_3, b_start_3, p); + *failures = true; + return; + } + } + + // Loook up end - 1 + HeapWord* addr_4 = the_end - 1; + HeapWord* b_start_4 = _offsets.block_start_const(addr_4); + if (b_start_4 != p) { + gclog_or_tty->print_cr("BOT look up for end - 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_4, b_start_4, p); + *failures = true; + return; } } @@ -880,12 +851,6 @@ "but has "SIZE_FORMAT", objects", bottom(), end(), object_num); *failures = true; - } - - if (p != top()) { - gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " - "does not match top "PTR_FORMAT, p, top()); - *failures = true; return; } } @@ -976,67 +941,3 @@ _offsets.set_space(this); initialize(mr, !is_zeroed, SpaceDecorator::Mangle); } - -size_t RegionList::length() { - size_t len = 0; - HeapRegion* cur = hd(); - DEBUG_ONLY(HeapRegion* last = NULL); - while (cur != NULL) { - len++; - DEBUG_ONLY(last = cur); - cur = get_next(cur); - } - assert(last == tl(), "Invariant"); - return len; -} - -void RegionList::insert_before_head(HeapRegion* r) { - assert(well_formed(), "Inv"); - set_next(r, hd()); - _hd = r; - _sz++; - if (tl() == NULL) _tl = r; - assert(well_formed(), "Inv"); -} - -void RegionList::prepend_list(RegionList* new_list) { - assert(well_formed(), "Precondition"); - assert(new_list->well_formed(), "Precondition"); - HeapRegion* new_tl = new_list->tl(); - if (new_tl != NULL) { - set_next(new_tl, hd()); - _hd = new_list->hd(); - _sz += new_list->sz(); - if (tl() == NULL) _tl = new_list->tl(); - } else { - assert(new_list->hd() == NULL && new_list->sz() == 0, "Inv"); - } - assert(well_formed(), "Inv"); -} - -void RegionList::delete_after(HeapRegion* r) { - assert(well_formed(), "Precondition"); - HeapRegion* next = get_next(r); - assert(r != NULL, "Precondition"); - HeapRegion* next_tl = get_next(next); - set_next(r, next_tl); - dec_sz(); - if (next == tl()) { - assert(next_tl == NULL, "Inv"); - _tl = r; - } - assert(well_formed(), "Inv"); -} - -HeapRegion* RegionList::pop() { - assert(well_formed(), "Inv"); - HeapRegion* res = hd(); - if (res != NULL) { - _hd = get_next(res); - _sz--; - set_next(res, NULL); - if (sz() == 0) _tl = NULL; - } - assert(well_formed(), "Inv"); - return res; -}
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp Fri Feb 18 11:15:40 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp Fri Feb 18 12:04:36 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,6 +50,11 @@ class HeapRegionRemSet; class HeapRegionRemSetIterator; class HeapRegion; +class HeapRegionSetBase; + +#define HR_FORMAT "%d:["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]" +#define HR_FORMAT_PARAMS(__hr) (__hr)->hrs_index(), (__hr)->bottom(), \ + (__hr)->top(), (__hr)->end() // A dirty card to oop closure for heap regions. It // knows how to get the G1 heap and how to use the bitmap @@ -173,6 +178,19 @@ virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); virtual void print() const; + + void reset_bot() { + _offsets.zero_bottom_entry(); + _offsets.initialize_threshold(); + } + + void update_bot_for_object(HeapWord* start, size_t word_size) { + _offsets.alloc_block(start, word_size); + } + + void print_bot_on(outputStream* out) { + _offsets.print_on(out); + } }; class HeapRegion: public G1OffsetTableContigSpace { @@ -214,12 +232,6 @@ // True iff the region is in current collection_set. bool _in_collection_set; - // True iff the region is on the unclean list, waiting to be zero filled. - bool _is_on_unclean_list; - - // True iff the region is on the free list, ready for allocation. - bool _is_on_free_list; - // Is this or has it been an allocation region in the current collection // pause. bool _is_gc_alloc_region; @@ -241,6 +253,13 @@ // Next region whose cards need cleaning HeapRegion* _next_dirty_cards_region; + // Fields used by the HeapRegionSetBase class and subclasses. + HeapRegion* _next; +#ifdef ASSERT + HeapRegionSetBase* _containing_set; +#endif // ASSERT + bool _pending_removal; + // For parallel heapRegion traversal. jint _claimed; @@ -292,10 +311,6 @@ _top_at_conc_mark_count = bot; } - jint _zfs; // A member of ZeroFillState. Protected by ZF_lock. - Thread* _zero_filler; // If _zfs is ZeroFilling, the thread that (last) - // made it so. - void set_young_type(YoungType new_type) { //assert(_young_type != new_type, "setting the same type" ); // TODO: add more assertions here @@ -349,15 +364,14 @@ RebuildRSClaimValue = 5 }; - // Concurrent refinement requires contiguous heap regions (in which TLABs - // might be allocated) to be zero-filled. Each region therefore has a - // zero-fill-state. - enum ZeroFillState { - NotZeroFilled, - ZeroFilling, - ZeroFilled, - Allocated - }; + inline HeapWord* par_allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return ContiguousSpace::par_allocate(word_size); + } + inline HeapWord* allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return ContiguousSpace::allocate(word_size); + } // If this region is a member of a HeapRegionSeq, the index in that // sequence, otherwise -1. @@ -404,13 +418,38 @@ return _humongous_start_region; } - // Causes the current region to represent a humongous object spanning "n" - // regions. - void set_startsHumongous(HeapWord* new_end); + // Makes the current region be a "starts humongous" region, i.e., + // the first region in a series of one or more contiguous regions + // that will contain a single "humongous" object. The two parameters + // are as follows: + // + // new_top : The new value of the top field of this region which + // points to the end of the humongous object that's being + // allocated. If there is more than one region in the series, top + // will lie beyond this region's original end field and on the last + // region in the series. + // + // new_end : The new value of the end field of this region which + // points to the end of the last region in the series. If there is + // one region in the series (namely: this one) end will be the same + // as the original end of this region. + // + // Updating top and end as described above makes this region look as + // if it spans the entire space taken up by all the regions in the + // series and an single allocation moved its top to new_top. This + // ensures that the space (capacity / allocated) taken up by all + // humongous regions can be calculated by just looking at the + // "starts humongous" regions and by ignoring the "continues + // humongous" regions. + void set_startsHumongous(HeapWord* new_top, HeapWord* new_end); - // The regions that continue a humongous sequence should be added using - // this method, in increasing address order. - void set_continuesHumongous(HeapRegion* start); + // Makes the current region be a "continues humongous' + // region. first_hr is the "start humongous" region of the series + // which this region will be part of. + void set_continuesHumongous(HeapRegion* first_hr); + + // Unsets the humongous-related fields on the region. + void set_notHumongous(); // If the region has a remembered set, return a pointer to it. HeapRegionRemSet* rem_set() const { @@ -458,45 +497,56 @@ _next_in_special_set = r; } - bool is_on_free_list() { - return _is_on_free_list; - } + // Methods used by the HeapRegionSetBase class and subclasses. - void set_on_free_list(bool b) { - _is_on_free_list = b; - } + // Getter and setter for the next field used to link regions into + // linked lists. + HeapRegion* next() { return _next; } + + void set_next(HeapRegion* next) { _next = next; } - HeapRegion* next_from_free_list() { - assert(is_on_free_list(), - "Should only invoke on free space."); - assert(_next_in_special_set == NULL || - _next_in_special_set->is_on_free_list(), - "Malformed Free List."); - return _next_in_special_set; - } + // Every region added to a set is tagged with a reference to that + // set. This is used for doing consistency checking to make sure that + // the contents of a set are as they should be and it's only + // available in non-product builds. +#ifdef ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { + assert((containing_set == NULL && _containing_set != NULL) || + (containing_set != NULL && _containing_set == NULL), + err_msg("containing_set: "PTR_FORMAT" " + "_containing_set: "PTR_FORMAT, + containing_set, _containing_set)); + + _containing_set = containing_set; +} - void set_next_on_free_list(HeapRegion* r) { - assert(r == NULL || r->is_on_free_list(), "Malformed free list."); - _next_in_special_set = r; - } + HeapRegionSetBase* containing_set() { return _containing_set; } +#else // ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { } - bool is_on_unclean_list() { - return _is_on_unclean_list; - } + // containing_set() is only used in asserts so there's not reason + // to provide a dummy version of it. +#endif // ASSERT - void set_on_unclean_list(bool b); + // If we want to remove regions from a list in bulk we can simply tag + // them with the pending_removal tag and call the + // remove_all_pending() method on the list. - HeapRegion* next_from_unclean_list() { - assert(is_on_unclean_list(), - "Should only invoke on unclean space."); - assert(_next_in_special_set == NULL || - _next_in_special_set->is_on_unclean_list(), - "Malformed unclean List."); - return _next_in_special_set; + bool pending_removal() { return _pending_removal; } + + void set_pending_removal(bool pending_removal) { + // We can only set pending_removal to true, if it's false and the + // region belongs to a set. + assert(!pending_removal || + (!_pending_removal && containing_set() != NULL), "pre-condition"); + // We can only set pending_removal to false, if it's true and the + // region does not belong to a set. + assert( pending_removal || + ( _pending_removal && containing_set() == NULL), "pre-condition"); + + _pending_removal = pending_removal; } - void set_next_on_unclean_list(HeapRegion* r); - HeapRegion* get_next_young_region() { return _next_young_region; } void set_next_young_region(HeapRegion* hr) { _next_young_region = hr; @@ -515,11 +565,6 @@ void initialize(MemRegion mr, bool clear_space, bool mangle_space); - // Ensure that "this" is zero-filled. - void ensure_zero_filled(); - // This one requires that the calling thread holds ZF_mon. - void ensure_zero_filled_locked(); - // Get the start of the unmarked area in this region. HeapWord* prev_top_at_mark_start() const { return _prev_top_at_mark_start; } HeapWord* next_top_at_mark_start() const { return _next_top_at_mark_start; } @@ -754,36 +799,6 @@ // "end" of the region if there is no such block. HeapWord* next_block_start_careful(HeapWord* addr); - // Returns the zero-fill-state of the current region. - ZeroFillState zero_fill_state() { return (ZeroFillState)_zfs; } - bool zero_fill_is_allocated() { return _zfs == Allocated; } - Thread* zero_filler() { return _zero_filler; } - - // Indicate that the contents of the region are unknown, and therefore - // might require zero-filling. - void set_zero_fill_needed() { - set_zero_fill_state_work(NotZeroFilled); - } - void set_zero_fill_in_progress(Thread* t) { - set_zero_fill_state_work(ZeroFilling); - _zero_filler = t; - } - void set_zero_fill_complete(); - void set_zero_fill_allocated() { - set_zero_fill_state_work(Allocated); - } - - void set_zero_fill_state_work(ZeroFillState zfs); - - // This is called when a full collection shrinks the heap. - // We want to set the heap region to a value which says - // it is no longer part of the heap. For now, we'll let "NotZF" fill - // that role. - void reset_zero_fill() { - set_zero_fill_state_work(NotZeroFilled); - _zero_filler = N