OpenJDK / jdk / jdk

changeset 57197:79fba633d07f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
8234543: ZGC: Parallel pre-touch Reviewed-by: eosterlund, stefank
author pliden
date Tue, 03 Dec 2019 13:51:29 +0100
parents 0821487ef0ab
children aa12d1f0bc66
files src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp src/hotspot/share/gc/z/zHeap.cpp src/hotspot/share/gc/z/zPageAllocator.cpp src/hotspot/share/gc/z/zPageAllocator.hpp src/hotspot/share/gc/z/zPhysicalMemory.cpp src/hotspot/share/gc/z/zPhysicalMemory.hpp src/hotspot/share/gc/z/zWorkers.cpp test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java
diffstat 13 files changed, 156 insertions(+), 43 deletions(-) [+]
line wrap: on
line diff
--- a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -148,7 +148,7 @@
   os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
 }
 
-void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const {
+void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
   const size_t nsegments = pmem.nsegments();
   size_t size = 0;
 
@@ -159,11 +159,6 @@
     _file.map(segment_addr, segment.size(), segment.start());
     size += segment.size();
   }
-
-  // Pre-touch memory
-  if (pretouch) {
-    pretouch_view(addr, size);
-  }
 }
 
 void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
@@ -175,15 +170,27 @@
   return ZAddress::marked0(offset);
 }
 
+void ZPhysicalMemoryBacking::pretouch(uintptr_t offset, size_t size) const {
+  if (ZVerifyViews) {
+    // Pre-touch good view
+    pretouch_view(ZAddress::good(offset), size);
+  } else {
+    // Pre-touch all views
+    pretouch_view(ZAddress::marked0(offset), size);
+    pretouch_view(ZAddress::marked1(offset), size);
+    pretouch_view(ZAddress::remapped(offset), size);
+  }
+}
+
 void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   if (ZVerifyViews) {
     // Map good view
-    map_view(pmem, ZAddress::good(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::good(offset));
   } else {
     // Map all views
-    map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::marked0(offset));
+    map_view(pmem, ZAddress::marked1(offset));
+    map_view(pmem, ZAddress::remapped(offset));
   }
 }
 
@@ -202,7 +209,7 @@
 void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   // Map good view
   assert(ZVerifyViews, "Should be enabled");
-  map_view(pmem, ZAddress::good(offset), false /* pretouch */);
+  map_view(pmem, ZAddress::good(offset));
 }
 
 void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const {
--- a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp	Tue Dec 03 13:51:29 2019 +0100
@@ -36,7 +36,7 @@
   ZMemoryManager _uncommitted;
 
   void pretouch_view(uintptr_t addr, size_t size) const;
-  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const;
+  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
   void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
 
 public:
@@ -53,6 +53,8 @@
 
   uintptr_t nmt_address(uintptr_t offset) const;
 
+  void pretouch(uintptr_t offset, size_t size) const;
+
   void map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
   void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
 
--- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -249,7 +249,7 @@
   os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
 }
 
-void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const {
+void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
   const size_t nsegments = pmem.nsegments();
   size_t size = 0;
 
@@ -273,11 +273,6 @@
 
   // NUMA interleave memory before touching it
   ZNUMA::memory_interleave(addr, size);
-
-  // Pre-touch memory
-  if (pretouch) {
-    pretouch_view(addr, size);
-  }
 }
 
 void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
@@ -296,15 +291,27 @@
   return ZAddress::marked0(offset);
 }
 
+void ZPhysicalMemoryBacking::pretouch(uintptr_t offset, size_t size) const {
+  if (ZVerifyViews) {
+    // Pre-touch good view
+    pretouch_view(ZAddress::good(offset), size);
+  } else {
+    // Pre-touch all views
+    pretouch_view(ZAddress::marked0(offset), size);
+    pretouch_view(ZAddress::marked1(offset), size);
+    pretouch_view(ZAddress::remapped(offset), size);
+  }
+}
+
 void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   if (ZVerifyViews) {
     // Map good view
-    map_view(pmem, ZAddress::good(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::good(offset));
   } else {
     // Map all views
-    map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::marked0(offset));
+    map_view(pmem, ZAddress::marked1(offset));
+    map_view(pmem, ZAddress::remapped(offset));
   }
 }
 
@@ -323,7 +330,7 @@
 void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   // Map good view
   assert(ZVerifyViews, "Should be enabled");
-  map_view(pmem, ZAddress::good(offset), false /* pretouch */);
+  map_view(pmem, ZAddress::good(offset));
 }
 
 void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const {
--- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp	Tue Dec 03 13:51:29 2019 +0100
@@ -43,7 +43,7 @@
 
   void advise_view(uintptr_t addr, size_t size, int advice) const;
   void pretouch_view(uintptr_t addr, size_t size) const;
-  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const;
+  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
   void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
 
 public:
@@ -60,6 +60,8 @@
 
   uintptr_t nmt_address(uintptr_t offset) const;
 
+  void pretouch(uintptr_t offset, size_t size) const;
+
   void map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
   void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
 
--- a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -149,7 +149,7 @@
   os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
 }
 
-void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const {
+void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
   const size_t nsegments = pmem.nsegments();
   size_t size = 0;
 
@@ -159,11 +159,6 @@
     _file.map(addr + size, segment.size(), segment.start());
     size += segment.size();
   }
-
-  // Pre-touch memory
-  if (pretouch) {
-    pretouch_view(addr, size);
-  }
 }
 
 void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
@@ -175,15 +170,27 @@
   return ZAddress::marked0(offset);
 }
 
+void ZPhysicalMemoryBacking::pretouch(uintptr_t offset, size_t size) const {
+  if (ZVerifyViews) {
+    // Pre-touch good view
+    pretouch_view(ZAddress::good(offset), size);
+  } else {
+    // Pre-touch all views
+    pretouch_view(ZAddress::marked0(offset), size);
+    pretouch_view(ZAddress::marked1(offset), size);
+    pretouch_view(ZAddress::remapped(offset), size);
+  }
+}
+
 void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   if (ZVerifyViews) {
     // Map good view
-    map_view(pmem, ZAddress::good(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::good(offset));
   } else {
     // Map all views
-    map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch);
-    map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch);
+    map_view(pmem, ZAddress::marked0(offset));
+    map_view(pmem, ZAddress::marked1(offset));
+    map_view(pmem, ZAddress::remapped(offset));
   }
 }
 
@@ -202,7 +209,7 @@
 void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   // Map good view
   assert(ZVerifyViews, "Should be enabled");
-  map_view(pmem, ZAddress::good(offset), false /* pretouch */);
+  map_view(pmem, ZAddress::good(offset));
 }
 
 void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const {
--- a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp	Tue Dec 03 13:51:29 2019 +0100
@@ -36,7 +36,7 @@
   ZMemoryManager _uncommitted;
 
   void pretouch_view(uintptr_t addr, size_t size) const;
-  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const;
+  void map_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
   void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
 
 public:
@@ -53,6 +53,8 @@
 
   uintptr_t nmt_address(uintptr_t offset) const;
 
+  void pretouch(uintptr_t offset, size_t size) const;
+
   void map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
   void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
 
--- a/src/hotspot/share/gc/z/zHeap.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zHeap.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -57,7 +57,7 @@
 ZHeap::ZHeap() :
     _workers(),
     _object_allocator(),
-    _page_allocator(heap_min_size(), heap_initial_size(), heap_max_size(), heap_max_reserve_size()),
+    _page_allocator(&_workers, heap_min_size(), heap_initial_size(), heap_max_size(), heap_max_reserve_size()),
     _page_table(),
     _forwarding_table(),
     _mark(&_workers, &_page_table),
--- a/src/hotspot/share/gc/z/zPageAllocator.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zPageAllocator.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -33,7 +33,9 @@
 #include "gc/z/zPageCache.inline.hpp"
 #include "gc/z/zSafeDelete.inline.hpp"
 #include "gc/z/zStat.hpp"
+#include "gc/z/zTask.hpp"
 #include "gc/z/zTracer.inline.hpp"
+#include "gc/z/zWorkers.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/init.hpp"
 #include "runtime/java.hpp"
@@ -95,7 +97,8 @@
 
 ZPage* const ZPageAllocator::gc_marker = (ZPage*)-1;
 
-ZPageAllocator::ZPageAllocator(size_t min_capacity,
+ZPageAllocator::ZPageAllocator(ZWorkers* workers,
+                               size_t min_capacity,
                                size_t initial_capacity,
                                size_t max_capacity,
                                size_t max_reserve) :
@@ -150,13 +153,42 @@
   }
 
   // Pre-map initial capacity
-  prime_cache(initial_capacity);
+  prime_cache(workers, initial_capacity);
 
   // Successfully initialized
   _initialized = true;
 }
 
-void ZPageAllocator::prime_cache(size_t size) {
+class ZPreTouchTask : public ZTask {
+private:
+  const ZPhysicalMemoryManager* const _physical;
+  volatile uintptr_t                  _start;
+  const uintptr_t                     _end;
+
+public:
+  ZPreTouchTask(const ZPhysicalMemoryManager* physical, uintptr_t start, uintptr_t end) :
+      ZTask("ZPreTouchTask"),
+      _physical(physical),
+      _start(start),
+      _end(end) {}
+
+  virtual void work() {
+    for (;;) {
+      // Get granule offset
+      const size_t size = ZGranuleSize;
+      const uintptr_t offset = Atomic::add(&_start, size) - size;
+      if (offset >= _end) {
+        // Done
+        break;
+      }
+
+      // Pre-touch granule
+      _physical->pretouch(offset, size);
+    }
+  }
+};
+
+void ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
   // Allocate physical memory
   const ZPhysicalMemory pmem = _physical.alloc(size);
   guarantee(!pmem.is_null(), "Invalid size");
@@ -172,6 +204,12 @@
   map_page(page);
   page->set_pre_mapped();
 
+  if (AlwaysPreTouch) {
+    // Pre-touch page
+    ZPreTouchTask task(&_physical, page->start(), page->end());
+    workers->run_parallel(&task);
+  }
+
   // Add page to cache
   page->set_last_used();
   _cache.free_page(page);
--- a/src/hotspot/share/gc/z/zPageAllocator.hpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zPageAllocator.hpp	Tue Dec 03 13:51:29 2019 +0100
@@ -34,6 +34,7 @@
 #include "memory/allocation.hpp"
 
 class ZPageAllocRequest;
+class ZWorkers;
 
 class ZPageAllocator {
   friend class VMStructs;
@@ -61,7 +62,7 @@
 
   static ZPage* const gc_marker;
 
-  void prime_cache(size_t size);
+  void prime_cache(ZWorkers* workers, size_t size);
 
   void increase_used(size_t size, bool relocation);
   void decrease_used(size_t size, bool reclaimed);
@@ -86,7 +87,8 @@
   void satisfy_alloc_queue();
 
 public:
-  ZPageAllocator(size_t min_capacity,
+  ZPageAllocator(ZWorkers* workers,
+                 size_t min_capacity,
                  size_t initial_capacity,
                  size_t max_capacity,
                  size_t max_reserve);
--- a/src/hotspot/share/gc/z/zPhysicalMemory.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zPhysicalMemory.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -173,6 +173,10 @@
   _backing.free(pmem);
 }
 
+void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const {
+  _backing.pretouch(offset, size);
+}
+
 void ZPhysicalMemoryManager::map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
   _backing.map(pmem, offset);
   nmt_commit(pmem, offset);
--- a/src/hotspot/share/gc/z/zPhysicalMemory.hpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zPhysicalMemory.hpp	Tue Dec 03 13:51:29 2019 +0100
@@ -82,6 +82,8 @@
   ZPhysicalMemory alloc(size_t size);
   void free(const ZPhysicalMemory& pmem);
 
+  void pretouch(uintptr_t offset, size_t size) const;
+
   void map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
   void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
 
--- a/src/hotspot/share/gc/z/zWorkers.cpp	Tue Dec 03 08:35:21 2019 -0400
+++ b/src/hotspot/share/gc/z/zWorkers.cpp	Tue Dec 03 13:51:29 2019 +0100
@@ -100,7 +100,6 @@
 }
 
 void ZWorkers::run_parallel(ZTask* task) {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint");
   run(task, nparallel());
 }
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java	Tue Dec 03 13:51:29 2019 +0100
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.z;
+
+/*
+ * @test TestAlwaysPreTouch
+ * @requires vm.gc.Z & !vm.graal.enabled
+ * @summary Test ZGC parallel pre-touch
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc* -XX:-AlwaysPreTouch -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ */
+
+public class TestAlwaysPreTouch {
+    public static void main(String[] args) throws Exception {
+        System.out.println("Success");
+    }
+}
© 2007, Oracle and/or its affiliates
Terms of Use · Privacy · Trademarks