OpenJDK / jdk / jdk
changeset 57580:ddc79542ce3a
8228818: Shenandoah: Processing weak roots in concurrent phase when possible
Reviewed-by: rkennke
line wrap: on
line diff
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -202,6 +202,7 @@ _heap(ShenandoahHeap::heap()) {} virtual void do_nmethod(nmethod* nm) { + assert(_heap->is_concurrent_root_in_progress(), "Only this phase"); if (failed()) { return; } @@ -222,9 +223,7 @@ ShenandoahReentrantLocker locker(nm_data->lock()); // Heal oops and disarm - if (_heap->is_evacuation_in_progress()) { - ShenandoahNMethod::heal_nmethod(nm); - } + ShenandoahNMethod::heal_nmethod(nm); ShenandoahNMethod::disarm_nmethod(nm); // Clear compiled ICs and exception caches
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -127,10 +127,12 @@ class ShenandoahUpdateRootsTask : public AbstractGangTask { private: ShenandoahRootUpdater* _root_updater; + bool _check_alive; public: - ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater) : + ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater, bool check_alive) : AbstractGangTask("Shenandoah update roots task"), - _root_updater(root_updater) { + _root_updater(root_updater), + _check_alive(check_alive){ } void work(uint worker_id) { @@ -139,8 +141,13 @@ ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahUpdateRefsClosure cl; - AlwaysTrueClosure always_true; - _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl); + if (_check_alive) { + ShenandoahForwardedIsAliveClosure is_alive; + _root_updater->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>(worker_id, &is_alive, &cl); + } else { + AlwaysTrueClosure always_true;; + _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl); + } } }; @@ -282,6 +289,8 @@ ShenandoahGCPhase phase(root_phase); + bool check_alive = root_phase == ShenandoahPhaseTimings::degen_gc_update_roots; + #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif @@ -289,7 +298,7 @@ uint nworkers = _heap->workers()->active_workers(); ShenandoahRootUpdater root_updater(nworkers, root_phase); - ShenandoahUpdateRootsTask update_roots(&root_updater); + ShenandoahUpdateRootsTask update_roots(&root_updater, check_alive); _heap->workers()->run_task(&update_roots); #if COMPILER2_OR_JVMCI
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -30,6 +30,7 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/locationPrinter.inline.hpp" #include "gc/shared/memAllocator.hpp" +#include "gc/shared/oopStorageSet.hpp" #include "gc/shared/plab.hpp" #include "gc/shenandoah/shenandoahAllocTracker.hpp" @@ -1651,20 +1652,25 @@ ShenandoahVMRoots<true /*concurrent*/> _vm_roots; ShenandoahWeakRoots<true /*concurrent*/> _weak_roots; ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots; + bool _include_weak_roots; public: - ShenandoahConcurrentRootsEvacUpdateTask() : - AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") { + ShenandoahConcurrentRootsEvacUpdateTask(bool include_weak_roots) : + AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task"), + _include_weak_roots(include_weak_roots) { } void work(uint worker_id) { ShenandoahEvacOOMScope oom; { - // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration + // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration // may race against OopStorage::release() calls. ShenandoahEvacUpdateOopStorageRootsClosure cl; _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl); - _weak_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl); + + if (_include_weak_roots) { + _weak_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl); + } } { @@ -1675,14 +1681,121 @@ } }; +class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure { +private: + ShenandoahHeap* const _heap; + ShenandoahMarkingContext* const _mark_context; + bool _evac_in_progress; + Thread* const _thread; + size_t _dead_counter; + +public: + ShenandoahEvacUpdateCleanupOopStorageRootsClosure(); + void do_oop(oop* p); + void do_oop(narrowOop* p); + + size_t dead_counter() const; + void reset_dead_counter(); +}; + +ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() : + _heap(ShenandoahHeap::heap()), + _mark_context(ShenandoahHeap::heap()->marking_context()), + _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()), + _thread(Thread::current()), + _dead_counter(0) { +} + +void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { + const oop obj = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(obj)) { + if (!_mark_context->is_marked(obj)) { + shenandoah_assert_correct(p, obj); + oop old = Atomic::cmpxchg(p, obj, oop(NULL)); + if (obj == old) { + _dead_counter ++; + } + assert(*p == NULL, "Must be"); + } else if (_evac_in_progress && _heap->in_collection_set(obj)) { + oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (resolved == obj) { + resolved = _heap->evacuate_object(obj, _thread); + } + Atomic::cmpxchg(p, obj, resolved); + assert(_heap->cancelled_gc() || + _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved), + "Sanity"); + } + } +} + +void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const { + return _dead_counter; +} + +void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() { + _dead_counter = 0; +} + +// This task not only evacuates/updates marked weak roots, but also "NULL" +// dead weak roots. +class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask { +private: + ShenandoahWeakRoot<true /*concurrent*/> _jni_roots; + ShenandoahWeakRoot<true /*concurrent*/> _string_table_roots; + ShenandoahWeakRoot<true /*concurrent*/> _resolved_method_table_roots; + ShenandoahWeakRoot<true /*concurrent*/> _vm_roots; + +public: + ShenandoahConcurrentWeakRootsEvacUpdateTask() : + AbstractGangTask("Shenandoah Concurrent Weak Root Task"), + _jni_roots(OopStorageSet::jni_weak(), ShenandoahPhaseTimings::JNIWeakRoots), + _string_table_roots(OopStorageSet::string_table_weak(), ShenandoahPhaseTimings::StringTableRoots), + _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), ShenandoahPhaseTimings::ResolvedMethodTableRoots), + _vm_roots(OopStorageSet::vm_weak(), ShenandoahPhaseTimings::VMWeakRoots) { + StringTable::reset_dead_counter(); + ResolvedMethodTable::reset_dead_counter(); + } + + ~ShenandoahConcurrentWeakRootsEvacUpdateTask() { + StringTable::finish_dead_counter(); + ResolvedMethodTable::finish_dead_counter(); + } + + void work(uint worker_id) { + ShenandoahEvacOOMScope oom; + // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration + // may race against OopStorage::release() calls. + ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl; + _jni_roots.oops_do(&cl, worker_id); + _vm_roots.oops_do(&cl, worker_id); + + cl.reset_dead_counter(); + _string_table_roots.oops_do(&cl, worker_id); + StringTable::inc_dead_counter(cl.dead_counter()); + + cl.reset_dead_counter(); + _resolved_method_table_roots.oops_do(&cl, worker_id); + ResolvedMethodTable::inc_dead_counter(cl.dead_counter()); + } +}; + void ShenandoahHeap::op_roots() { if (is_concurrent_root_in_progress()) { if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { + // Concurrent weak root processing + ShenandoahConcurrentWeakRootsEvacUpdateTask task; + workers()->run_task(&task); + _unloader.unload(); } if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) { - ShenandoahConcurrentRootsEvacUpdateTask task; + ShenandoahConcurrentRootsEvacUpdateTask task(!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); workers()->run_task(&task); } } @@ -2076,32 +2189,34 @@ ShenandoahPhaseTimings::purge_par; // Cleanup weak roots ShenandoahGCPhase phase(timing_phase); + phase_timings()->record_workers_start(timing_phase); if (has_forwarded_objects()) { if (is_traversal_mode()) { ShenandoahForwardedIsAliveClosure is_alive; ShenandoahTraversalUpdateRefsClosure keep_alive; ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahTraversalUpdateRefsClosure> - cleaning_task(&is_alive, &keep_alive, num_workers); + cleaning_task(&is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); _workers->run_task(&cleaning_task); } else { ShenandoahForwardedIsAliveClosure is_alive; ShenandoahUpdateRefsClosure keep_alive; ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure> - cleaning_task(&is_alive, &keep_alive, num_workers); + cleaning_task(&is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); _workers->run_task(&cleaning_task); } } else { ShenandoahIsAliveClosure is_alive; #ifdef ASSERT - ShenandoahAssertNotForwardedClosure verify_cl; - ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure> - cleaning_task(&is_alive, &verify_cl, num_workers); + ShenandoahAssertNotForwardedClosure verify_cl; + ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure> + cleaning_task(&is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); #else - ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure> - cleaning_task(&is_alive, &do_nothing_cl, num_workers); + ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure> + cleaning_task(&is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()); #endif _workers->run_task(&cleaning_task); } + phase_timings()->record_workers_end(timing_phase); } void ShenandoahHeap::parallel_cleaning(bool full_gc) { @@ -2348,7 +2463,7 @@ if (ShenandoahVerify) { if (!is_degenerated_gc_in_progress()) { - verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots); + verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots); } verifier()->verify_before_updaterefs(); }
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -28,6 +28,7 @@ #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shenandoah/shenandoahForwarding.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" @@ -73,6 +74,12 @@ Universe::verify(); } + // Degenerated GC may carry concurrent_root_in_progress flag when upgrading to + // full GC. We need to reset it before mutators resume. + if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + heap->set_concurrent_root_in_progress(false); + } + heap->set_full_gc_in_progress(true); assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
--- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -170,6 +170,7 @@ } void ShenandoahNMethod::heal_nmethod(nmethod* nm) { + assert(ShenandoahHeap::heap()->is_concurrent_root_in_progress(), "Only this phase"); ShenandoahNMethod* data = gc_data(nm); assert(data != NULL, "Sanity"); assert(data->lock()->owned_by_self(), "Must hold the lock");
--- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -54,13 +54,15 @@ } void ShenandoahNMethod::disarm_nmethod(nmethod* nm) { - if (!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { - return; - } + if (!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + return; + } - BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); - assert(bs != NULL, "Sanity"); - bs->disarm(nm); + BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + assert(bs != NULL, "Sanity"); + if (bs->is_armed(nm)) { + bs->disarm(nm); + } } ShenandoahNMethod* ShenandoahNMethod::gc_data(nmethod* nm) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahNormalMode.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahNormalMode.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -35,6 +35,7 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahSuspendibleWorkers); + SHENANDOAH_ERGO_DISABLE_FLAG(VerifyBeforeExit); } // Final configuration checks
--- a/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.hpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.hpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -28,17 +28,24 @@ #include "gc/shared/weakProcessor.hpp" #include "gc/shared/weakProcessorPhaseTimes.hpp" #include "gc/shared/workgroup.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" #include "memory/iterator.hpp" // Perform weak root cleaning at a pause template <typename IsAlive, typename KeepAlive> class ShenandoahParallelWeakRootsCleaningTask : public AbstractGangTask { protected: - WeakProcessor::Task _weak_processing_task; - IsAlive* _is_alive; - KeepAlive* _keep_alive; + WeakProcessor::Task _weak_processing_task; + ShenandoahSerialWeakRoots _serial_weak_roots; + IsAlive* _is_alive; + KeepAlive* _keep_alive; + bool _include_concurrent_roots; + public: - ShenandoahParallelWeakRootsCleaningTask(IsAlive* is_alive, KeepAlive* keep_alive, uint num_workers); + ShenandoahParallelWeakRootsCleaningTask(IsAlive* is_alive, + KeepAlive* keep_alive, + uint num_workers, + bool include_concurrent_roots); ~ShenandoahParallelWeakRootsCleaningTask(); void work(uint worker_id);
--- a/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.inline.hpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.inline.hpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -34,10 +34,11 @@ template<typename IsAlive, typename KeepAlive> ShenandoahParallelWeakRootsCleaningTask<IsAlive, KeepAlive>::ShenandoahParallelWeakRootsCleaningTask(IsAlive* is_alive, KeepAlive* keep_alive, - uint num_workers) : + uint num_workers, + bool include_concurrent_roots) : AbstractGangTask("Parallel Weak Root Cleaning Task"), _weak_processing_task(num_workers), - _is_alive(is_alive), _keep_alive(keep_alive) { + _is_alive(is_alive), _keep_alive(keep_alive), _include_concurrent_roots(include_concurrent_roots) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); if (ShenandoahStringDedup::is_enabled()) { @@ -54,7 +55,11 @@ template<typename IsAlive, typename KeepAlive> void ShenandoahParallelWeakRootsCleaningTask<IsAlive, KeepAlive>::work(uint worker_id) { - _weak_processing_task.work<IsAlive, KeepAlive>(worker_id, _is_alive, _keep_alive); + if (_include_concurrent_roots) { + _weak_processing_task.work<IsAlive, KeepAlive>(worker_id, _is_alive, _keep_alive); + } else { + _serial_weak_roots.weak_oops_do(_is_alive, _keep_alive, worker_id); + } if (ShenandoahStringDedup::is_enabled()) { ShenandoahStringDedup::parallel_oops_do(_is_alive, _keep_alive, worker_id);
--- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -90,6 +90,8 @@ phase == final_update_refs_roots || phase == full_gc_roots || phase == degen_gc_update_roots || + phase == full_gc_purge_par || + phase == purge_par || phase == _num_phases, "only in these phases we can add per-thread phase times"); if (phase != _num_phases) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -51,7 +51,7 @@ } bool ShenandoahRootVerifier::verify(RootTypes type) const { - return (_types & type) != 0; + return (_types & type) == type; } ShenandoahRootVerifier::RootTypes ShenandoahRootVerifier::combine(RootTypes t1, RootTypes t2) { @@ -89,6 +89,11 @@ shenandoah_assert_safepoint(); AlwaysTrueClosure always_true; WeakProcessor::weak_oops_do(&always_true, oops); + } else if (verify(SerialWeakRoots)) { + shenandoah_assert_safepoint(); + serial_weak_roots_do(oops); + } else if (verify(ConcurrentWeakRoots)) { + concurrent_weak_roots_do(oops); } if (ShenandoahStringDedup::is_enabled() && verify(StringDedupRoots)) { @@ -154,3 +159,18 @@ // dangling reference from the thread root. Threads::possibly_parallel_oops_do(false, oops, &blobs); } + +void ShenandoahRootVerifier::serial_weak_roots_do(OopClosure* cl) { + WeakProcessorPhases::Iterator itr = WeakProcessorPhases::serial_iterator(); + AlwaysTrueClosure always_true; + for ( ; !itr.is_end(); ++itr) { + WeakProcessorPhases::processor(*itr)(&always_true, cl); + } +} + +void ShenandoahRootVerifier::concurrent_weak_roots_do(OopClosure* cl) { + for (OopStorageSet::Iterator it = OopStorageSet::weak_iterator(); !it.is_end(); ++it) { + OopStorage* storage = *it; + storage->oops_do<OopClosure>(cl); + } +}
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -35,9 +35,11 @@ ThreadRoots = 1 << 1, CodeRoots = 1 << 2, CLDGRoots = 1 << 3, - WeakRoots = 1 << 4, - StringDedupRoots = 1 << 5, - JNIHandleRoots = 1 << 6, + SerialWeakRoots = 1 << 4, + ConcurrentWeakRoots = 1 << 5, + WeakRoots = (SerialWeakRoots | ConcurrentWeakRoots), + StringDedupRoots = 1 << 6, + JNIHandleRoots = 1 << 7, AllRoots = (SerialRoots | ThreadRoots | CodeRoots | CLDGRoots | WeakRoots | StringDedupRoots | JNIHandleRoots) }; @@ -57,6 +59,9 @@ static RootTypes combine(RootTypes t1, RootTypes t2); private: bool verify(RootTypes type) const; + + void serial_weak_roots_do(OopClosure* cl); + void concurrent_weak_roots_do(OopClosure* cl); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHROOTVERIFIER_HPP
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahForwarding.inline.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" @@ -632,7 +633,8 @@ VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, - VerifyGCState gcstate) { + VerifyGCState gcstate, + VerifyWeakRoots weak_roots) { guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); @@ -724,6 +726,18 @@ size_t count_reachable = 0; if (ShenandoahVerifyLevel >= 2) { ShenandoahRootVerifier verifier; + switch (weak_roots) { + case _verify_serial_weak_roots: + verifier.excludes(ShenandoahRootVerifier::ConcurrentWeakRoots); + break; + case _verify_concurrent_weak_roots: + verifier.excludes(ShenandoahRootVerifier::SerialWeakRoots); + break; + case _verify_all_weak_roots: + break; + default: + ShouldNotReachHere(); + } ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &verifier, label, options); _heap->workers()->run_task(&task); @@ -791,7 +805,8 @@ _verify_cset_disable, // cset may be inconsistent _verify_liveness_disable, // no reliable liveness data _verify_regions_disable, // no reliable region data - _verify_gcstate_disable // no data about gcstate + _verify_gcstate_disable, // no data about gcstate + _verify_all_weak_roots ); } @@ -804,7 +819,8 @@ _verify_cset_forwarded, // allow forwarded references to cset _verify_liveness_disable, // no reliable liveness data _verify_regions_notrash, // no trash regions - _verify_gcstate_forwarded // there are forwarded objects + _verify_gcstate_forwarded, // there are forwarded objects + _verify_all_weak_roots ); } else { verify_at_safepoint( @@ -814,7 +830,8 @@ _verify_cset_none, // UR should have fixed this _verify_liveness_disable, // no reliable liveness data _verify_regions_notrash, // no trash regions - _verify_gcstate_stable // there are no forwarded objects + _verify_gcstate_stable, // there are no forwarded objects + _verify_all_weak_roots ); } } @@ -827,11 +844,17 @@ _verify_cset_none, // no references to cset anymore _verify_liveness_complete, // liveness data must be complete here _verify_regions_disable, // trash regions not yet recycled - _verify_gcstate_stable // mark should have stabilized the heap + _verify_gcstate_stable, // mark should have stabilized the heap + _verify_all_weak_roots ); } void ShenandoahVerifier::verify_before_evacuation() { + // Concurrent weak roots are evacuated during concurrent phase + VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ? + _verify_serial_weak_roots : + _verify_all_weak_roots; + verify_at_safepoint( "Before Evacuation", _verify_forwarded_none, // no forwarded references @@ -839,19 +862,26 @@ _verify_cset_disable, // non-forwarded references to cset expected _verify_liveness_complete, // liveness data must be complete here _verify_regions_disable, // trash regions not yet recycled - _verify_gcstate_stable // mark should have stabilized the heap + _verify_gcstate_stable, // mark should have stabilized the heap + verify_weak_roots ); } void ShenandoahVerifier::verify_during_evacuation() { + // Concurrent weak roots are evacuated during concurrent phase + VerifyWeakRoots verify_weak_roots = ShenandoahConcurrentRoots::should_do_concurrent_class_unloading() ? + _verify_serial_weak_roots : + _verify_all_weak_roots; + verify_at_safepoint( "During Evacuation", - _verify_forwarded_allow, // some forwarded references are allowed - _verify_marked_disable, // walk only roots - _verify_cset_disable, // some cset references are not forwarded yet - _verify_liveness_disable, // liveness data might be already stale after pre-evacs - _verify_regions_disable, // trash regions not yet recycled - _verify_gcstate_evacuation // evacuation is in progress + _verify_forwarded_allow, // some forwarded references are allowed + _verify_marked_disable, // walk only roots + _verify_cset_disable, // some cset references are not forwarded yet + _verify_liveness_disable, // liveness data might be already stale after pre-evacs + _verify_regions_disable, // trash regions not yet recycled + _verify_gcstate_evacuation, // evacuation is in progress + verify_weak_roots ); } @@ -863,7 +893,8 @@ _verify_cset_forwarded, // all cset refs are fully forwarded _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash, // trash regions have been recycled already - _verify_gcstate_forwarded // evacuation produced some forwarded objects + _verify_gcstate_forwarded, // evacuation produced some forwarded objects + _verify_all_weak_roots ); } @@ -875,7 +906,8 @@ _verify_cset_forwarded, // all cset refs are fully forwarded _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash, // trash regions have been recycled already - _verify_gcstate_forwarded // evacuation should have produced some forwarded objects + _verify_gcstate_forwarded, // evacuation should have produced some forwarded objects + _verify_all_weak_roots ); } @@ -887,7 +919,8 @@ _verify_cset_none, // no cset references, all updated _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_nocset, // no cset regions, trash regions have appeared - _verify_gcstate_stable // update refs had cleaned up forwarded objects + _verify_gcstate_stable, // update refs had cleaned up forwarded objects + _verify_all_weak_roots ); } @@ -899,7 +932,8 @@ _verify_cset_none, // no cset references _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash_nocset, // no trash, no cset - _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects + _verify_gcstate_stable, // degenerated refs had cleaned up forwarded objects + _verify_all_weak_roots ); } @@ -911,7 +945,8 @@ _verify_cset_none, // no cset references before traversal _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash_nocset, // no trash and no cset regions - _verify_gcstate_stable // nothing forwarded before traversal + _verify_gcstate_stable, // nothing forwarded before traversal + _verify_all_weak_roots ); } @@ -923,7 +958,8 @@ _verify_cset_none, // no cset references left after traversal _verify_liveness_disable, // liveness data is not collected for new allocations _verify_regions_nocset, // no cset regions, trash regions allowed - _verify_gcstate_stable // nothing forwarded after traversal + _verify_gcstate_stable, // nothing forwarded after traversal + _verify_all_weak_roots ); } @@ -935,7 +971,8 @@ _verify_cset_disable, // cset might be foobared _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_disable, // no reliable region data here - _verify_gcstate_disable // no reliable gcstate data + _verify_gcstate_disable, // no reliable gcstate data + _verify_all_weak_roots ); } @@ -947,7 +984,8 @@ _verify_cset_none, // no cset references _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash_nocset, // no trash, no cset - _verify_gcstate_stable // full gc cleaned up everything + _verify_gcstate_stable, // full gc cleaned up everything + _verify_all_weak_roots ); }
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Thu Jan 09 02:27:20 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Thu Jan 09 08:35:44 2020 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -141,6 +141,12 @@ _verify_gcstate_evacuation } VerifyGCState; + typedef enum { + _verify_all_weak_roots, + _verify_serial_weak_roots, + _verify_concurrent_weak_roots + } VerifyWeakRoots; + struct VerifyOptions { VerifyForwarded _verify_forwarded; VerifyMarked _verify_marked; @@ -148,17 +154,20 @@ VerifyLiveness _verify_liveness; VerifyRegions _verify_regions; VerifyGCState _verify_gcstate; + VerifyWeakRoots _verify_weak_roots; VerifyOptions(VerifyForwarded verify_forwarded, VerifyMarked verify_marked, VerifyCollectionSet verify_collection_set, VerifyLiveness verify_liveness, VerifyRegions verify_regions, - VerifyGCState verify_gcstate) : + VerifyGCState verify_gcstate, + VerifyWeakRoots verify_weak_roots = _verify_all_weak_roots) : _verify_forwarded(verify_forwarded), _verify_marked(verify_marked), _verify_cset(verify_collection_set), _verify_liveness(verify_liveness), _verify_regions(verify_regions), - _verify_gcstate(verify_gcstate) {} + _verify_gcstate(verify_gcstate), + _verify_weak_roots(verify_weak_roots) {} }; private: @@ -168,7 +177,8 @@ VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, - VerifyGCState gcstate); + VerifyGCState gcstate, + VerifyWeakRoots weakRoots); public: ShenandoahVerifier(ShenandoahHeap* heap, MarkBitMap* verification_bitmap) :