changeset 57249:366c0f357ee6

8212160: JVMTI agent crashes with "assert(_value != 0LL) failed: resolving NULL _value" Summary: Add local deferred event list to thread to post events outside CodeCache_lock. Reviewed-by: eosterlund, dholmes, sspitsyn
author coleenp
date Thu, 05 Dec 2019 16:57:17 -0500
parents 7d732f6e17b2
children fcd70fd2d3f6
files src/hotspot/share/code/nmethod.cpp src/hotspot/share/code/nmethod.hpp src/hotspot/share/jfr/leakprofiler/checkpoint/rootResolver.cpp src/hotspot/share/prims/jvmtiCodeBlobEvents.cpp src/hotspot/share/prims/jvmtiExport.cpp src/hotspot/share/prims/jvmtiExport.hpp src/hotspot/share/prims/jvmtiImpl.cpp src/hotspot/share/prims/jvmtiImpl.hpp src/hotspot/share/prims/jvmtiThreadState.cpp src/hotspot/share/prims/jvmtiThreadState.hpp src/hotspot/share/runtime/serviceThread.cpp src/hotspot/share/runtime/serviceThread.hpp src/hotspot/share/runtime/thread.cpp test/hotspot/jtreg/serviceability/jvmti/CompiledMethodLoad/Zombie.java test/hotspot/jtreg/serviceability/jvmti/CompiledMethodLoad/libCompiledZombie.cpp
diffstat 15 files changed, 299 insertions(+), 92 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/share/code/nmethod.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/code/nmethod.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -49,6 +49,7 @@
 #include "oops/methodData.hpp"
 #include "oops/oop.inline.hpp"
 #include "prims/jvmtiImpl.hpp"
+#include "prims/jvmtiThreadState.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/flags/flagSetting.hpp"
@@ -58,6 +59,7 @@
 #include "runtime/orderAccess.hpp"
 #include "runtime/os.hpp"
 #include "runtime/safepointVerifiers.hpp"
+#include "runtime/serviceThread.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/sweeper.hpp"
 #include "runtime/vmThread.hpp"
@@ -428,7 +430,8 @@
   _has_flushed_dependencies   = 0;
   _lock_count                 = 0;
   _stack_traversal_mark       = 0;
-  _unload_reported            = false; // jvmti state
+  _load_reported              = false; // jvmti state
+  _unload_reported            = false;
   _is_far_code                = false; // nmethods are located in CodeCache
 
 #ifdef ASSERT
@@ -436,7 +439,6 @@
 #endif
 
   _oops_do_mark_link       = NULL;
-  _jmethod_id              = NULL;
   _osr_link                = NULL;
 #if INCLUDE_RTM_OPT
   _rtm_state               = NoRTM;
@@ -1563,11 +1565,11 @@
 // post_compiled_method_load_event
 // new method for install_code() path
 // Transfer information from compilation to jvmti
-void nmethod::post_compiled_method_load_event() {
-
- // This is a bad time for a safepoint.  We don't want
- // this nmethod to get unloaded while we're queueing the event.
- NoSafepointVerifier nsv;
+void nmethod::post_compiled_method_load_event(JvmtiThreadState* state) {
+
+  // This is a bad time for a safepoint.  We don't want
+  // this nmethod to get unloaded while we're queueing the event.
+  NoSafepointVerifier nsv;
 
   Method* m = method();
   HOTSPOT_COMPILED_METHOD_LOAD(
@@ -1579,28 +1581,24 @@
       m->signature()->utf8_length(),
       insts_begin(), insts_size());
 
-  if (JvmtiExport::should_post_compiled_method_load() ||
-      JvmtiExport::should_post_compiled_method_unload()) {
-    get_and_cache_jmethod_id();
-  }
 
   if (JvmtiExport::should_post_compiled_method_load()) {
-    // Let the Service thread (which is a real Java thread) post the event
-    MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
-    JvmtiDeferredEventQueue::enqueue(
-      JvmtiDeferredEvent::compiled_method_load_event(this));
+    // Only post unload events if load events are found.
+    set_load_reported();
+    // Keep sweeper from turning this into zombie until it is posted.
+    mark_as_seen_on_stack();
+
+    // If a JavaThread hasn't been passed in, let the Service thread
+    // (which is a real Java thread) post the event
+    JvmtiDeferredEvent event = JvmtiDeferredEvent::compiled_method_load_event(this);
+    if (state == NULL) {
+      ServiceThread::enqueue_deferred_event(&event);
+    } else {
+      state->enqueue_event(&event);
+    }
   }
 }
 
-jmethodID nmethod::get_and_cache_jmethod_id() {
-  if (_jmethod_id == NULL) {
-    // Cache the jmethod_id since it can no longer be looked up once the
-    // method itself has been marked for unloading.
-    _jmethod_id = method()->jmethod_id();
-  }
-  return _jmethod_id;
-}
-
 void nmethod::post_compiled_method_unload() {
   if (unload_reported()) {
     // During unloading we transition to unloaded and then to zombie
@@ -1614,17 +1612,17 @@
   // If a JVMTI agent has enabled the CompiledMethodUnload event then
   // post the event. Sometime later this nmethod will be made a zombie
   // by the sweeper but the Method* will not be valid at that point.
-  // If the _jmethod_id is null then no load event was ever requested
-  // so don't bother posting the unload.  The main reason for this is
-  // that the jmethodID is a weak reference to the Method* so if
+  // The jmethodID is a weak reference to the Method* so if
   // it's being unloaded there's no way to look it up since the weak
   // ref will have been cleared.
-  if (_jmethod_id != NULL && JvmtiExport::should_post_compiled_method_unload()) {
+
+  // Don't bother posting the unload if the load event wasn't posted.
+  if (load_reported() && JvmtiExport::should_post_compiled_method_unload()) {
     assert(!unload_reported(), "already unloaded");
     JvmtiDeferredEvent event =
-      JvmtiDeferredEvent::compiled_method_unload_event(_jmethod_id, insts_begin());
-    MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
-    JvmtiDeferredEventQueue::enqueue(event);
+      JvmtiDeferredEvent::compiled_method_unload_event(
+          method()->jmethod_id(), insts_begin());
+    ServiceThread::enqueue_deferred_event(&event);
   }
 
   // The JVMTI CompiledMethodUnload event can be enabled or disabled at
--- a/src/hotspot/share/code/nmethod.hpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/code/nmethod.hpp	Thu Dec 05 16:57:17 2019 -0500
@@ -30,6 +30,7 @@
 class DepChange;
 class DirectiveSet;
 class DebugInformationRecorder;
+class JvmtiThreadState;
 
 // nmethods (native methods) are the compiled code versions of Java methods.
 //
@@ -71,7 +72,6 @@
  private:
   // Shared fields for all nmethod's
   int       _entry_bci;        // != InvocationEntryBci if this nmethod is an on-stack replacement method
-  jmethodID _jmethod_id;       // Cache of method()->jmethod_id()
 
   // To support simple linked-list chaining of nmethods:
   nmethod*  _osr_link;         // from InstanceKlass::osr_nmethods_head
@@ -227,8 +227,9 @@
   // protected by CodeCache_lock
   bool _has_flushed_dependencies;            // Used for maintenance of dependencies (CodeCache_lock)
 
-  // used by jvmti to track if an unload event has been posted for this nmethod.
+  // used by jvmti to track if an event has been posted for this nmethod.
   bool _unload_reported;
+  bool _load_reported;
 
   // Protected by CompiledMethod_lock
   volatile signed char _state;               // {not_installed, in_use, not_entrant, zombie, unloaded}
@@ -482,10 +483,6 @@
   bool  make_not_used()    { return make_not_entrant(); }
   bool  make_zombie()      { return make_not_entrant_or_zombie(zombie); }
 
-  // used by jvmti to track if the unload event has been reported
-  bool  unload_reported()                         { return _unload_reported; }
-  void  set_unload_reported()                     { _unload_reported = true; }
-
   int get_state() const {
     return _state;
   }
@@ -621,6 +618,12 @@
 
   address* orig_pc_addr(const frame* fr);
 
+  // used by jvmti to track if the load and unload events has been reported
+  bool  unload_reported() const                   { return _unload_reported; }
+  void  set_unload_reported()                     { _unload_reported = true; }
+  bool  load_reported() const                     { return _load_reported; }
+  void  set_load_reported()                       { _load_reported = true; }
+
  public:
   // copying of debugging information
   void copy_scopes_pcs(PcDesc* pcs, int count);
@@ -631,8 +634,7 @@
   void    set_original_pc(const frame* fr, address pc) { *orig_pc_addr(fr) = pc; }
 
   // jvmti support:
-  void post_compiled_method_load_event();
-  jmethodID get_and_cache_jmethod_id();
+  void post_compiled_method_load_event(JvmtiThreadState* state = NULL);
 
   // verify operations
   void verify();
--- a/src/hotspot/share/jfr/leakprofiler/checkpoint/rootResolver.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/rootResolver.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -370,7 +370,7 @@
 
   JvmtiThreadState* const jvmti_thread_state = jt->jvmti_thread_state();
   if (jvmti_thread_state != NULL) {
-    jvmti_thread_state->oops_do(&rcl);
+    jvmti_thread_state->oops_do(&rcl, NULL);
   }
 
   return rcl.complete();
--- a/src/hotspot/share/prims/jvmtiCodeBlobEvents.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiCodeBlobEvents.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -32,6 +32,7 @@
 #include "oops/oop.inline.hpp"
 #include "prims/jvmtiCodeBlobEvents.hpp"
 #include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/vmThread.hpp"
 
@@ -219,25 +220,27 @@
 
 // Generate a COMPILED_METHOD_LOAD event for each nnmethod
 jvmtiError JvmtiCodeBlobEvents::generate_compiled_method_load_events(JvmtiEnv* env) {
-  HandleMark hm;
+  JvmtiThreadState* state = JvmtiThreadState::state_for(JavaThread::current());
+  {
+    // Walk the CodeCache notifying for live nmethods, don't release the CodeCache_lock
+    // because the sweeper may be running concurrently.
+    // Save events to the queue for posting outside the CodeCache_lock.
+    MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+    // Iterate over non-profiled and profiled nmethods
+    NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);
+    while(iter.next()) {
+      nmethod* current = iter.method();
+      current->post_compiled_method_load_event(state);
+    }
+  }
 
-  // Walk the CodeCache notifying for live nmethods.  The code cache
-  // may be changing while this is happening which is ok since newly
-  // created nmethod will notify normally and nmethods which are freed
-  // can be safely skipped.
-  MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
-  // Iterate over non-profiled and profiled nmethods
-  NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);
-  while(iter.next()) {
-    nmethod* current = iter.method();
-    // Lock the nmethod so it can't be freed
-    nmethodLocker nml(current);
-
-    // Don't hold the lock over the notify or jmethodID creation
-    MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
-    current->get_and_cache_jmethod_id();
-    JvmtiExport::post_compiled_method_load(env, current);
-  }
+  // Now post all the events outside the CodeCache_lock.
+  // If there's a safepoint, the queued events will be kept alive.
+  // Adding these events to the service thread to post is something that
+  // should work, but the service thread doesn't keep up in stress scenarios and
+  // the os eventually kills the process with OOM.
+  // We want this thread to wait until the events are all posted.
+  state->post_events(env);
   return JVMTI_ERROR_NONE;
 }
 
--- a/src/hotspot/share/prims/jvmtiExport.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiExport.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -59,6 +59,7 @@
 #include "runtime/objectMonitor.inline.hpp"
 #include "runtime/os.inline.hpp"
 #include "runtime/safepointVerifiers.hpp"
+#include "runtime/serviceThread.hpp"
 #include "runtime/thread.inline.hpp"
 #include "runtime/threadSMR.hpp"
 #include "runtime/vframe.inline.hpp"
@@ -1352,11 +1353,10 @@
 
   // postings to the service thread so that it can perform them in a safe
   // context and in-order.
-  MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
   ResourceMark rm;
   // JvmtiDeferredEvent copies the string.
   JvmtiDeferredEvent event = JvmtiDeferredEvent::class_unload_event(klass->name()->as_C_string());
-  JvmtiDeferredEventQueue::enqueue(event);
+  ServiceThread::enqueue_deferred_event(&event);
 }
 
 
@@ -2235,10 +2235,9 @@
     // It may not be safe to post the event from this thread.  Defer all
     // postings to the service thread so that it can perform them in a safe
     // context and in-order.
-    MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
     JvmtiDeferredEvent event = JvmtiDeferredEvent::dynamic_code_generated_event(
         name, code_begin, code_end);
-    JvmtiDeferredEventQueue::enqueue(event);
+    ServiceThread::enqueue_deferred_event(&event);
   }
 }
 
--- a/src/hotspot/share/prims/jvmtiExport.hpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiExport.hpp	Thu Dec 05 16:57:17 2019 -0500
@@ -167,7 +167,6 @@
   // DynamicCodeGenerated events for a given environment.
   friend class JvmtiCodeBlobEvents;
 
-  static void post_compiled_method_load(JvmtiEnv* env, nmethod *nm) NOT_JVMTI_RETURN;
   static void post_dynamic_code_generated(JvmtiEnv* env, const char *name, const void *code_begin,
                                           const void *code_end) NOT_JVMTI_RETURN;
 
@@ -342,6 +341,7 @@
                                         unsigned char **data_ptr, unsigned char **end_ptr,
                                         JvmtiCachedClassFileData **cache_ptr) NOT_JVMTI_RETURN_(false);
   static void post_native_method_bind(Method* method, address* function_ptr) NOT_JVMTI_RETURN;
+  static void post_compiled_method_load(JvmtiEnv* env, nmethod *nm) NOT_JVMTI_RETURN;
   static void post_compiled_method_load(nmethod *nm) NOT_JVMTI_RETURN;
   static void post_dynamic_code_generated(const char *name, const void *code_begin, const void *code_end) NOT_JVMTI_RETURN;
 
--- a/src/hotspot/share/prims/jvmtiImpl.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiImpl.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -986,6 +986,13 @@
   }
 }
 
+void JvmtiDeferredEvent::post_compiled_method_load_event(JvmtiEnv* env) {
+  assert(_type == TYPE_COMPILED_METHOD_LOAD, "only user of this method");
+  nmethod* nm = _event_data.compiled_method_load;
+  JvmtiExport::post_compiled_method_load(env, nm);
+}
+
+
 // Keep the nmethod for compiled_method_load from being unloaded.
 void JvmtiDeferredEvent::oops_do(OopClosure* f, CodeBlobClosure* cf) {
   if (cf != NULL && _type == TYPE_COMPILED_METHOD_LOAD) {
@@ -998,20 +1005,14 @@
 void JvmtiDeferredEvent::nmethods_do(CodeBlobClosure* cf) {
   if (cf != NULL && _type == TYPE_COMPILED_METHOD_LOAD) {
     cf->do_code_blob(_event_data.compiled_method_load);
-  }  // May add UNLOAD event but it doesn't work yet.
+  }
 }
 
-JvmtiDeferredEventQueue::QueueNode* JvmtiDeferredEventQueue::_queue_tail = NULL;
-JvmtiDeferredEventQueue::QueueNode* JvmtiDeferredEventQueue::_queue_head = NULL;
-
 bool JvmtiDeferredEventQueue::has_events() {
-  assert(Service_lock->owned_by_self(), "Must own Service_lock");
   return _queue_head != NULL;
 }
 
-void JvmtiDeferredEventQueue::enqueue(const JvmtiDeferredEvent& event) {
-  assert(Service_lock->owned_by_self(), "Must own Service_lock");
-
+void JvmtiDeferredEventQueue::enqueue(JvmtiDeferredEvent event) {
   // Events get added to the end of the queue (and are pulled off the front).
   QueueNode* node = new QueueNode(event);
   if (_queue_tail == NULL) {
@@ -1022,14 +1023,11 @@
     _queue_tail = node;
   }
 
-  Service_lock->notify_all();
   assert((_queue_head == NULL) == (_queue_tail == NULL),
          "Inconsistent queue markers");
 }
 
 JvmtiDeferredEvent JvmtiDeferredEventQueue::dequeue() {
-  assert(Service_lock->owned_by_self(), "Must own Service_lock");
-
   assert(_queue_head != NULL, "Nothing to dequeue");
 
   if (_queue_head == NULL) {
@@ -1051,6 +1049,14 @@
   return event;
 }
 
+void JvmtiDeferredEventQueue::post(JvmtiEnv* env) {
+  // Post and destroy queue nodes
+  while (_queue_head != NULL) {
+     JvmtiDeferredEvent event = dequeue();
+     event.post_compiled_method_load_event(env);
+  }
+}
+
 void JvmtiDeferredEventQueue::oops_do(OopClosure* f, CodeBlobClosure* cf) {
   for(QueueNode* node = _queue_head; node != NULL; node = node->next()) {
      node->event().oops_do(f, cf);
--- a/src/hotspot/share/prims/jvmtiImpl.hpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiImpl.hpp	Thu Dec 05 16:57:17 2019 -0500
@@ -480,6 +480,7 @@
 
   // Actually posts the event.
   void post() NOT_JVMTI_RETURN;
+  void post_compiled_method_load_event(JvmtiEnv* env) NOT_JVMTI_RETURN;
   // Sweeper support to keep nmethods from being zombied while in the queue.
   void nmethods_do(CodeBlobClosure* cf) NOT_JVMTI_RETURN;
   // GC support to keep nmethod from being unloaded while in the queue.
@@ -491,7 +492,7 @@
  * and posts the events.  The Service_lock is required to be held
  * when operating on the queue.
  */
-class JvmtiDeferredEventQueue : AllStatic {
+class JvmtiDeferredEventQueue : public CHeapObj<mtInternal> {
   friend class JvmtiDeferredEvent;
  private:
   class QueueNode : public CHeapObj<mtInternal> {
@@ -509,18 +510,23 @@
     void set_next(QueueNode* next) { _next = next; }
   };
 
-  static QueueNode* _queue_head;             // Hold Service_lock to access
-  static QueueNode* _queue_tail;             // Hold Service_lock to access
+  QueueNode* _queue_head;
+  QueueNode* _queue_tail;
 
  public:
-  // Must be holding Service_lock when calling these
-  static bool has_events() NOT_JVMTI_RETURN_(false);
-  static void enqueue(const JvmtiDeferredEvent& event) NOT_JVMTI_RETURN;
-  static JvmtiDeferredEvent dequeue() NOT_JVMTI_RETURN_(JvmtiDeferredEvent());
+  JvmtiDeferredEventQueue() : _queue_head(NULL), _queue_tail(NULL) {}
+
+  bool has_events() NOT_JVMTI_RETURN_(false);
+  JvmtiDeferredEvent dequeue() NOT_JVMTI_RETURN_(JvmtiDeferredEvent());
+
+  // Post all events in the queue for the current Jvmti environment
+  void post(JvmtiEnv* env) NOT_JVMTI_RETURN_(false);
+  void enqueue(JvmtiDeferredEvent event) NOT_JVMTI_RETURN;
+
   // Sweeper support to keep nmethods from being zombied while in the queue.
-  static void nmethods_do(CodeBlobClosure* cf) NOT_JVMTI_RETURN;
+  void nmethods_do(CodeBlobClosure* cf) NOT_JVMTI_RETURN;
   // GC support to keep nmethod from being unloaded while in the queue.
-  static void oops_do(OopClosure* f, CodeBlobClosure* cf) NOT_JVMTI_RETURN;
+  void oops_do(OopClosure* f, CodeBlobClosure* cf) NOT_JVMTI_RETURN;
 };
 
 // Utility macro that checks for NULL pointers:
--- a/src/hotspot/share/prims/jvmtiThreadState.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiThreadState.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,8 @@
   _earlyret_value.j = 0L;
   _earlyret_oop = NULL;
 
+  _jvmti_event_queue = NULL;
+
   // add all the JvmtiEnvThreadState to the new JvmtiThreadState
   {
     JvmtiEnvIterator it;
@@ -397,6 +399,36 @@
   }
 }
 
-void JvmtiThreadState::oops_do(OopClosure* f) {
+void JvmtiThreadState::oops_do(OopClosure* f, CodeBlobClosure* cf) {
   f->do_oop((oop*) &_earlyret_oop);
+
+  // Keep nmethods from unloading on the event queue
+  if (_jvmti_event_queue != NULL) {
+    _jvmti_event_queue->oops_do(f, cf);
+  }
 }
+
+void JvmtiThreadState::nmethods_do(CodeBlobClosure* cf) {
+  // Keep nmethods from unloading on the event queue
+  if (_jvmti_event_queue != NULL) {
+    _jvmti_event_queue->nmethods_do(cf);
+  }
+}
+
+// Thread local event queue.
+void JvmtiThreadState::enqueue_event(JvmtiDeferredEvent* event) {
+  if (_jvmti_event_queue == NULL) {
+    _jvmti_event_queue = new JvmtiDeferredEventQueue();
+  }
+  // copy the event
+  _jvmti_event_queue->enqueue(*event);
+}
+
+void JvmtiThreadState::post_events(JvmtiEnv* env) {
+  if (_jvmti_event_queue != NULL) {
+    _jvmti_event_queue->post(env);  // deletes each queue node
+    delete _jvmti_event_queue;
+    _jvmti_event_queue = NULL;
+  }
+}
+
--- a/src/hotspot/share/prims/jvmtiThreadState.hpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/prims/jvmtiThreadState.hpp	Thu Dec 05 16:57:17 2019 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,9 @@
 class JvmtiEnvThreadState;
 class JvmtiDynamicCodeEventCollector;
 
+class JvmtiDeferredEvent;
+class JvmtiDeferredEventQueue;
+
 enum JvmtiClassLoadKind {
   jvmti_class_load_kind_load = 100,
   jvmti_class_load_kind_retransform,
@@ -75,6 +78,8 @@
  private:
   friend class JvmtiEnv;
   JavaThread        *_thread;
+  // Jvmti Events that cannot be posted in their current context.
+  JvmtiDeferredEventQueue* _jvmti_event_queue;
   bool              _hide_single_stepping;
   bool              _pending_step_for_popframe;
   bool              _pending_step_for_earlyret;
@@ -384,10 +389,15 @@
   static ByteSize earlyret_oop_offset()   { return byte_offset_of(JvmtiThreadState, _earlyret_oop); }
   static ByteSize earlyret_value_offset() { return byte_offset_of(JvmtiThreadState, _earlyret_value); }
 
-  void oops_do(OopClosure* f) NOT_JVMTI_RETURN; // GC support
+  void oops_do(OopClosure* f, CodeBlobClosure* cf) NOT_JVMTI_RETURN; // GC support
+  void nmethods_do(CodeBlobClosure* cf) NOT_JVMTI_RETURN;
 
 public:
   void set_should_post_on_exceptions(bool val) { _thread->set_should_post_on_exceptions_flag(val ? JNI_TRUE : JNI_FALSE); }
+
+  // Thread local event queue, which doesn't require taking the Service_lock.
+  void enqueue_event(JvmtiDeferredEvent* event);
+  void post_events(JvmtiEnv* env);
 };
 
 class RedefineVerifyMark : public StackObj {
--- a/src/hotspot/share/runtime/serviceThread.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/runtime/serviceThread.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -47,6 +47,9 @@
 
 ServiceThread* ServiceThread::_instance = NULL;
 JvmtiDeferredEvent* ServiceThread::_jvmti_event = NULL;
+// The service thread has it's own static deferred event queue.
+// Events can be posted before the service thread is created.
+JvmtiDeferredEventQueue ServiceThread::_jvmti_service_queue;
 
 void ServiceThread::initialize() {
   EXCEPTION_MARK;
@@ -124,7 +127,7 @@
       // tests from potentially starving later work.  Hence the use of
       // arithmetic-or to combine results; we don't want short-circuiting.
       while (((sensors_changed = (!UseNotificationThread && LowMemoryDetector::has_pending_requests())) |
-              (has_jvmti_events = JvmtiDeferredEventQueue::has_events()) |
+              (has_jvmti_events = _jvmti_service_queue.has_events()) |
               (has_gc_notification_event = (!UseNotificationThread && GCNotifier::has_event())) |
               (has_dcmd_notification_event = (!UseNotificationThread && DCmdFactory::has_pending_jmx_notification())) |
               (stringtable_work = StringTable::has_work()) |
@@ -140,7 +143,7 @@
 
       if (has_jvmti_events) {
         // Get the event under the Service_lock
-        jvmti_event = JvmtiDeferredEventQueue::dequeue();
+        jvmti_event = _jvmti_service_queue.dequeue();
         _jvmti_event = &jvmti_event;
       }
     }
@@ -190,6 +193,12 @@
   }
 }
 
+void ServiceThread::enqueue_deferred_event(JvmtiDeferredEvent* event) {
+  MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
+  _jvmti_service_queue.enqueue(*event);
+  Service_lock->notify_all();
+ }
+
 void ServiceThread::oops_do(OopClosure* f, CodeBlobClosure* cf) {
   JavaThread::oops_do(f, cf);
   // The ServiceThread "owns" the JVMTI Deferred events, scan them here
@@ -198,8 +207,9 @@
     if (_jvmti_event != NULL) {
       _jvmti_event->oops_do(f, cf);
     }
+    // Requires a lock, because threads can be adding to this queue.
     MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
-    JvmtiDeferredEventQueue::oops_do(f, cf);
+    _jvmti_service_queue.oops_do(f, cf);
   }
 }
 
@@ -209,7 +219,8 @@
     if (_jvmti_event != NULL) {
       _jvmti_event->nmethods_do(cf);
     }
+    // Requires a lock, because threads can be adding to this queue.
     MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
-    JvmtiDeferredEventQueue::nmethods_do(cf);
+    _jvmti_service_queue.nmethods_do(cf);
   }
 }
--- a/src/hotspot/share/runtime/serviceThread.hpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/runtime/serviceThread.hpp	Thu Dec 05 16:57:17 2019 -0500
@@ -25,6 +25,7 @@
 #ifndef SHARE_RUNTIME_SERVICETHREAD_HPP
 #define SHARE_RUNTIME_SERVICETHREAD_HPP
 
+#include "prims/jvmtiImpl.hpp"
 #include "runtime/thread.hpp"
 
 // A hidden from external view JavaThread for JVMTI compiled-method-load
@@ -37,6 +38,7 @@
  private:
   static ServiceThread* _instance;
   static JvmtiDeferredEvent* _jvmti_event;
+  static JvmtiDeferredEventQueue _jvmti_service_queue;
 
   static void service_thread_entry(JavaThread* thread, TRAPS);
   ServiceThread(ThreadFunction entry_point) : JavaThread(entry_point) {};
@@ -48,6 +50,9 @@
   bool is_hidden_from_external_view() const      { return true; }
   bool is_service_thread() const                 { return true; }
 
+  // Add event to the service thread event queue.
+  static void enqueue_deferred_event(JvmtiDeferredEvent* event);
+
   // GC support
   void oops_do(OopClosure* f, CodeBlobClosure* cf);
   void nmethods_do(CodeBlobClosure* cf);
--- a/src/hotspot/share/runtime/thread.cpp	Thu Dec 05 13:10:18 2019 -0800
+++ b/src/hotspot/share/runtime/thread.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -2995,7 +2995,7 @@
   f->do_oop((oop*) &_pending_async_exception);
 
   if (jvmti_thread_state() != NULL) {
-    jvmti_thread_state()->oops_do(f);
+    jvmti_thread_state()->oops_do(f, cf);
   }
 }
 
@@ -3021,6 +3021,10 @@
       fst.current()->nmethods_do(cf);
     }
   }
+
+  if (jvmti_thread_state() != NULL) {
+    jvmti_thread_state()->nmethods_do(cf);
+  }
 }
 
 void JavaThread::metadata_do(MetadataClosure* f) {
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/jvmti/CompiledMethodLoad/Zombie.java	Thu Dec 05 16:57:17 2019 -0500
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8212159
+ * @summary Generate compiled method load events without crashing
+ * @run main/othervm/native -agentlib:CompiledZombie -Xcomp -XX:ReservedCodeCacheSize=20m Zombie
+ *
+ * The stress test that made this fail was -jar SwingSet2.jar from demos (without DISPLAY set so it exits)
+ */
+
+public class Zombie {
+    public static void main(java.lang.String[] unused) {
+        // There are plenty of compiled methods with -Xcomp even without doing much.
+        System.out.println("Test passes if it doesn't crash while posting compiled method events.");
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/jvmti/CompiledMethodLoad/libCompiledZombie.cpp	Thu Dec 05 16:57:17 2019 -0500
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "jvmti.h"
+#include "jni.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static int events;
+static int total_events = 0;
+
+void JNICALL CompiledMethodLoad(jvmtiEnv* jvmti, jmethodID method,
+                                jint code_size, const void* code_addr,
+                                jint map_length, const jvmtiAddrLocationMap* map,
+                                const void* compile_info) {
+    events++;
+    total_events++;
+}
+
+// Continuously generate CompiledMethodLoad events for all currently compiled methods
+void JNICALL GenerateEventsThread(jvmtiEnv* jvmti, JNIEnv* jni, void* arg) {
+    jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_COMPILED_METHOD_LOAD, NULL);
+    int count = 0;
+
+    while (true) {
+        events = 0;
+        jvmti->GenerateEvents(JVMTI_EVENT_COMPILED_METHOD_LOAD);
+        if (events != 0 && ++count == 200) {
+            printf("Generated %d events\n", events);
+            count = 0;
+        }
+    }
+}
+
+// As soon as VM starts, run a separate Agent thread that will generate CompiledMethodLoad events
+void JNICALL VMInit(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread) {
+    jclass thread_class = jni->FindClass("java/lang/Thread");
+    jmethodID thread_constructor = jni->GetMethodID(thread_class, "<init>", "()V");
+    jthread agent_thread = jni->NewObject(thread_class, thread_constructor);
+
+    jvmti->RunAgentThread(agent_thread, GenerateEventsThread, NULL, JVMTI_THREAD_NORM_PRIORITY);
+}
+
+jint Agent_OnLoad(JavaVM* vm, char* options, void* reserved) {
+    jvmtiEnv* jvmti;
+    vm->GetEnv((void**)&jvmti, JVMTI_VERSION_1_0);
+
+    jvmtiCapabilities capabilities;
+    memset(&capabilities, 0, sizeof(capabilities));
+
+    capabilities.can_generate_compiled_method_load_events = 1;
+    jvmti->AddCapabilities(&capabilities);
+
+    jvmtiEventCallbacks callbacks;
+    memset(&callbacks, 0, sizeof(callbacks));
+    callbacks.VMInit = VMInit;
+    callbacks.CompiledMethodLoad = CompiledMethodLoad;
+    jvmti->SetEventCallbacks(&callbacks, sizeof(callbacks));
+    jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_INIT, NULL);
+
+    return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif
+