feat(vt): expose carrier frames and fix truncated traces for virtual threads on JDK 21+

ddprof-lib/src/main/cpp/arguments.cpp

@@ -292,9 +292,11 @@ Error Arguments::parse(const char *args) {
         } else if (strcmp(value, "vm") == 0) {
           _cstack = CSTACK_VM;
         } else if (strcmp(value, "vmx") == 0) {
-          // cstack=vmx is a shorthand for cstack=vm,features=mixed
+          // cstack=vmx is a shorthand for cstack=vm,features=mixed; carrier-frame
+          // unwinding is enabled automatically since vmx already traverses entry frames
           _cstack = CSTACK_VM;
           _features.mixed = 1;
+          _features.carrier_frames = 1;
         } else {
           _cstack = CSTACK_NO;
         }

ddprof-lib/src/main/cpp/arguments.h

@@ -122,7 +122,8 @@ struct StackWalkFeatures {
     unsigned short vtable_target : 1;  // show receiver classes of vtable/itable stubs
     unsigned short comp_task     : 1;  // display current compilation task for JIT threads
     unsigned short pc_addr       : 1;  // record exact PC address for each sample
-    unsigned short _padding      : 3;  // pad structure to 16 bits
+    unsigned short carrier_frames: 1;  // walk through VT continuation boundary to carrier frames (enabled automatically with cstack=vmx)
+    unsigned short _padding      : 2;  // pad structure to 16 bits
 };
 
 struct Multiplier {

ddprof-lib/src/main/cpp/counters.h

@@ -92,7 +92,11 @@
   X(WALKVM_STUB_FRAMESIZE_FALLBACK, "walkvm_stub_framesize_fallback")        \
   X(WALKVM_FP_CHAIN_ATTEMPT, "walkvm_fp_chain_attempt")                      \
   X(WALKVM_FP_CHAIN_REACHED_CODEHEAP, "walkvm_fp_chain_reached_codeheap")    \
-  X(WALKVM_ANCHOR_NOT_IN_JAVA, "walkvm_anchor_not_in_java")                    \
+  X(WALKVM_ANCHOR_NOT_IN_JAVA,  "walkvm_anchor_not_in_java")                   \
+  X(WALKVM_CONT_BARRIER_HIT,    "walkvm_cont_barrier_hit")                     \
+  X(WALKVM_ENTER_SPECIAL_HIT,   "walkvm_enter_special_hit")                    \
+  X(WALKVM_CONT_SPECULATIVE_HIT,"walkvm_cont_speculative_hit")                 \
+  X(WALKVM_CONT_ENTRY_NULL,     "walkvm_cont_entry_null")                      \
   X(NATIVE_LIBS_DROPPED, "native_libs_dropped")                                \
   X(SIGACTION_PATCHED_LIBS, "sigaction_patched_libs")                          \
   X(SIGACTION_INTERCEPTED, "sigaction_intercepted")

ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp

@@ -35,6 +35,9 @@ int VMStructs::_narrow_klass_shift = -1;
 int VMStructs::_interpreter_frame_bcp_offset = 0;
 unsigned char VMStructs::_unsigned5_base = 0;
 const void* VMStructs::_call_stub_return = nullptr;
+const void* VMStructs::_cont_return_barrier   = nullptr;
+const void* VMStructs::_cont_entry_return_pc  = nullptr;
+VMNMethod*  VMStructs::_enter_special_nm       = nullptr;
 const void* VMStructs::_interpreter_start = nullptr;
 VMNMethod* VMStructs::_interpreter_nm = nullptr;
 const void* VMStructs::_interpreted_frame_valid_start = nullptr;
@@ -44,6 +47,7 @@ const void* VMStructs::_interpreted_frame_valid_end = nullptr;
 // Initialize type size to 0
 #define INIT_TYPE_SIZE(name, names) uint64_t VMStructs::TYPE_SIZE_NAME(name) = 0;
 DECLARE_TYPES_DO(INIT_TYPE_SIZE)
+DECLARE_V21_TYPES_DO(INIT_TYPE_SIZE)
 #undef INIT_TYPE_SIZE
 
 #define offset_value -1
@@ -62,6 +66,7 @@ DECLARE_TYPES_DO(INIT_TYPE_SIZE)
     field_type VMStructs::var = field_type##_value;
 
 DECLARE_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING)
+DECLARE_V21_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING)
 
 #undef INIT_FIELD
 #undef INIT_FIELD_WITH_VERSION
@@ -175,6 +180,7 @@ void VMStructs::init_offsets_and_addresses() {
 
 #define END_TYPE() continue; }
         DECLARE_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE)
+        DECLARE_V21_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE)
 #undef MATCH_TYPE_NAMES
 #undef READ_FIELD_VALUE
 #undef READ_FIELD_VALUE_WITH_VERSION
@@ -205,6 +211,7 @@ void VMStructs::init_type_sizes() {
         }
 
         DECLARE_TYPES_DO(READ_TYPE_SIZE)
+        DECLARE_V21_TYPES_DO(READ_TYPE_SIZE)
 
 #undef READ_TYPE_SIZE   
 
@@ -271,12 +278,21 @@ void VMStructs::verify_offsets() {
     }
 
 // Verify type sizes
+// Note: DECLARE_V21_TYPES_DO (VMContinuationEntry) is intentionally excluded here.
+// ContinuationEntry is not exported in gHotSpotVMTypes before JDK 27 (added via JDK-8378985);
+// asserting type_size() > 0 would SIGABRT on any JDK 21-26 build.
 #define VERIFY_TYPE_SIZE(name, names) assert(TYPE_SIZE_NAME(name) > 0);
     DECLARE_TYPES_DO(VERIFY_TYPE_SIZE);
 #undef VERIFY_TYPE_SIZE
 
 
 // Verify offsets and addresses
+// Note: DECLARE_V21_TYPE_FIELD_DO is intentionally excluded here.
+// Continuation-related fields (_cont_entry_offset, _cont_return_barrier_addr,
+// _cont_entry_return_pc_addr, _cont_entry_parent_offset) are absent from
+// gHotSpotVMStructs in all JDK 21-26 builds: ContinuationEntry was not
+// exported in the vmStructs table until JDK 27 (JDK-8378985). walkVM degrades
+// gracefully when they are missing.
 #define offset_value -1
 #define address_value nullptr
 
@@ -391,6 +407,25 @@ void VMStructs::resolveOffsets() {
     if (_call_stub_return_addr != NULL) {
         _call_stub_return = *(const void**)_call_stub_return_addr;
     }
+    if (_cont_return_barrier_addr != NULL) {
+        _cont_return_barrier = *(const void**)_cont_return_barrier_addr;
+    }
+    if (_cont_entry_return_pc_addr != NULL) {
+        _cont_entry_return_pc = *(const void**)_cont_entry_return_pc_addr;
+    }
+    // Fallback for JDK 21-26: StubRoutines::_cont_returnBarrier and
+    // ContinuationEntry::_return_pc are absent from gHotSpotVMStructs before
+    // JDK 27 (added via JDK-8378985).  Resolve them via C++ symbol lookup.
+    // Symbol names use Itanium C++ ABI mangling (GCC/Clang), which matches
+    // the HotSpot build toolchain on all supported platforms.
+    if (_cont_return_barrier == nullptr && VM::hotspot_version() >= 21) {
+        const void** sym = (const void**)_libjvm->findSymbol("_ZN12StubRoutines19_cont_returnBarrierE");
+        if (sym != nullptr) _cont_return_barrier = *sym;
+    }
+    if (_cont_entry_return_pc == nullptr && VM::hotspot_version() >= 21) {
+        const void** sym = (const void**)_libjvm->findSymbol("_ZN17ContinuationEntry10_return_pcE");
+        if (sym != nullptr) _cont_entry_return_pc = *sym;
+    }
 
     // Since JDK 23, _metadata_offset is relative to _data_offset. See metadata()
     if (_nmethod_immutable_offset < 0) {
@@ -440,6 +475,14 @@ void VMStructs::resolveOffsets() {
     if (_interpreter_nm == NULL && _interpreter_start != NULL) {
         _interpreter_nm = CodeHeap::findNMethod(_interpreter_start);
     }
+    if (_enter_special_nm == NULL && _cont_entry_return_pc != NULL) {
+        // On JDK 27+, enterSpecial is a proper nmethod; findNMethod succeeds.
+        // On JDK 21-26, it is a RuntimeBlob; findNMethod returns NULL and
+        // _enter_special_nm stays NULL.  The cont_entry_return_pc boundary is
+        // then detected via isContEntryReturnPc() in the walk loop rather than
+        // nmethod identity.
+        _enter_special_nm = CodeHeap::findNMethod(_cont_entry_return_pc);
+    }
 }
 
 void VMStructs::initJvmFunctions() {

ddprof-lib/src/main/cpp/hotspot/vmStructs.h

@@ -116,16 +116,38 @@ inline T* cast_to(const void* ptr) {
  */
 
 #define DECLARE_TYPES_DO(f) \
-    f(VMClassLoaderData,    MATCH_SYMBOLS("ClassLoaderData"))   \
-    f(VMConstantPool,       MATCH_SYMBOLS("ConstantPool"))      \
-    f(VMConstMethod,        MATCH_SYMBOLS("ConstMethod"))       \
-    f(VMFlag,               MATCH_SYMBOLS("JVMFlag", "Flag"))   \
-    f(VMJavaFrameAnchor,    MATCH_SYMBOLS("JavaFrameAnchor"))   \
-    f(VMKlass,              MATCH_SYMBOLS("Klass"))             \
-    f(VMMethod,             MATCH_SYMBOLS("Method"))            \
-    f(VMNMethod,            MATCH_SYMBOLS("nmethod"))           \
-    f(VMSymbol,             MATCH_SYMBOLS("Symbol"))            \
-    f(VMThread,             MATCH_SYMBOLS("Thread"))
+    f(VMClassLoaderData,      MATCH_SYMBOLS("ClassLoaderData"))   \
+    f(VMConstantPool,         MATCH_SYMBOLS("ConstantPool"))      \
+    f(VMConstMethod,          MATCH_SYMBOLS("ConstMethod"))       \
+    f(VMFlag,                 MATCH_SYMBOLS("JVMFlag", "Flag"))   \
+    f(VMJavaFrameAnchor,      MATCH_SYMBOLS("JavaFrameAnchor"))   \
+    f(VMKlass,                MATCH_SYMBOLS("Klass"))             \
+    f(VMMethod,               MATCH_SYMBOLS("Method"))            \
+    f(VMNMethod,              MATCH_SYMBOLS("nmethod"))           \
+    f(VMSymbol,               MATCH_SYMBOLS("Symbol"))            \
+    f(VMThread,               MATCH_SYMBOLS("Thread"))
+
+// Types only present in JDK 21+ (Project Loom); size is 0 on older JDKs
+#define DECLARE_V21_TYPES_DO(f) \
+    f(VMContinuationEntry,    MATCH_SYMBOLS("ContinuationEntry"))
+
+// Fields for JDK 21+ virtual-thread / continuation support.
+// ContinuationEntry was not exported in gHotSpotVMStructs until JDK 27
+// (JDK-8378985), so these fields are absent from the table in all JDK 21-26
+// builds and are populated via C++ mangled-symbol fallback instead.  They are
+// intentionally excluded from verify_offsets() so that a missing entry causes
+// graceful degradation rather than SIGABRT.
+#define DECLARE_V21_TYPE_FIELD_DO(type_begin, field, field_with_version, type_end) \
+    type_begin(VMJavaThread, MATCH_SYMBOLS("JavaThread", "Thread"))                \
+        field_with_version(_cont_entry_offset, offset, 21, MAX_VERSION, MATCH_SYMBOLS("_cont_entry")) \
+    type_end()                                                                     \
+    type_begin(VMStubRoutine, MATCH_SYMBOLS("StubRoutines"))                      \
+        field_with_version(_cont_return_barrier_addr, address, 21, MAX_VERSION, MATCH_SYMBOLS("_cont_returnBarrier")) \
+    type_end()                                                                     \
+    type_begin(VMContinuationEntry, MATCH_SYMBOLS("ContinuationEntry"))           \
+        field_with_version(_cont_entry_return_pc_addr, address, 21, MAX_VERSION, MATCH_SYMBOLS("_return_pc")) \
+        field_with_version(_cont_entry_parent_offset,  offset,  21, MAX_VERSION, MATCH_SYMBOLS("_parent"))    \
+    type_end()
 
 /**
  * Following macros define field offsets, addresses or values of JVM classes that are exported by
@@ -250,7 +272,7 @@ typedef void* address;
         field(_vs_high_offset, offset, MATCH_SYMBOLS("_high"))                                                      \
     type_end()                                                                                                      \
     type_begin(VMStubRoutine, MATCH_SYMBOLS("StubRoutines"))                                                        \
-        field(_call_stub_return_addr, address, MATCH_SYMBOLS("_call_stub_return_address"))                          \
+        field(_call_stub_return_addr, address, MATCH_SYMBOLS("_call_stub_return_address"))                         \
     type_end()                                                                                                      \
     type_begin(VMGrowableArray, MATCH_SYMBOLS("GrowableArrayBase", "GenericGrowableArray"))                         \
         field(_array_len_offset, offset, MATCH_SYMBOLS("_len"))                                                     \
@@ -313,6 +335,9 @@ class VMStructs {
     static int _interpreter_frame_bcp_offset;
     static unsigned char _unsigned5_base;
     static const void* _call_stub_return;
+    static const void* _cont_return_barrier;
+    static const void* _cont_entry_return_pc;
+    static VMNMethod*  _enter_special_nm;
     static const void* _interpreter_start;
     static VMNMethod* _interpreter_nm;
     static const void* _interpreted_frame_valid_start;
@@ -324,6 +349,7 @@ class VMStructs {
     static uint64_t TYPE_SIZE_NAME(name);
 
     DECLARE_TYPES_DO(DECLARE_TYPE_SIZE_VAR)
+    DECLARE_V21_TYPES_DO(DECLARE_TYPE_SIZE_VAR)
 #undef DECLARE_TYPE_SIZE_VAR
 
 // Declare vmStructs' field offsets and addresses
@@ -336,6 +362,7 @@ class VMStructs {
     static field_type var;
 
     DECLARE_TYPE_FIELD_DO(DO_NOTHING, DECLARE_TYPE_FIELD, DECLARE_TYPE_FIELD_WITH_VERSION, DO_NOTHING)
+    DECLARE_V21_TYPE_FIELD_DO(DO_NOTHING, DECLARE_TYPE_FIELD, DECLARE_TYPE_FIELD_WITH_VERSION, DO_NOTHING)
 #undef DECLARE_TYPE_FIELD
 #undef DECLARE_TYPE_FIELD_WITH_VERSION
 #undef DO_NOTHING
@@ -448,6 +475,21 @@ class VMStructs {
         return pc >= _interpreted_frame_valid_start && pc < _interpreted_frame_valid_end;
     }
 
+    static bool isContReturnBarrier(const void* pc) {
+        return _cont_return_barrier != nullptr && pc == _cont_return_barrier;
+    }
+
+    // True when the bottom VT frame's return PC is cont_entry_return_pc, meaning all
+    // VT frames are thawed (CPU-bound VT that never yielded).
+    // Available on JDK 21+ via vmStructs or symbol fallback.
+    static bool isContEntryReturnPc(const void* pc) {
+        return _cont_entry_return_pc != nullptr && pc == _cont_entry_return_pc;
+    }
+
+    static VMNMethod* enterSpecialNMethod() {
+        return _enter_special_nm;
+    }
+
     // Datadog-specific extensions
     static bool isSafeToWalk(uintptr_t pc);
     static void JNICALL NativeMethodBind(jvmtiEnv *jvmti, JNIEnv *jni,
@@ -675,6 +717,30 @@ DECLARE(VMJavaFrameAnchor)
     }
 DECLARE_END
 
+DECLARE(VMContinuationEntry)
+  public:
+    // Address of the enterSpecial frame's {saved_fp, return_addr} pair.
+    // Layout above this address: [saved_fp][return_addr_to_carrier][carrier_sp...]
+    // The ContinuationEntry struct is embedded on the carrier stack immediately
+    // below enterSpecial's saved-fp slot; its size() equals the JVM's
+    // ContinuationEntry::size() static method, confirmed at:
+    //   https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.hpp
+    //   https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.cpp
+    uintptr_t entryFP() const {
+        assert(type_size() > 0); // must not be called before ContinuationEntry is resolved
+        return (uintptr_t)this + type_size();
+    }
+
+    // Returns the enclosing ContinuationEntry when continuations are nested
+    // (e.g. a Continuation.run() call inside a virtual thread).  Returns
+    // nullptr when there is no enclosing entry or the field is unavailable.
+    VMContinuationEntry* parent() const {
+        if (_cont_entry_parent_offset < 0) return nullptr;
+        void* ptr = SafeAccess::loadPtr((void**) const_cast<VMContinuationEntry*>(this)->at(_cont_entry_parent_offset), nullptr);
+        return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr;
+    }
+DECLARE_END
+
 // Copied from JDK's globalDefinitions.hpp 'JavaThreadState' enum
 enum JVMJavaThreadState {
     _thread_uninitialized     =  0, // should never happen (missing initialization)
@@ -793,6 +859,27 @@ DECLARE(VMThread)
         return VMJavaFrameAnchor::cast(at(_thread_anchor_offset));
     }
 
+    // Returns true when this thread is currently executing a virtual thread
+    // (i.e. JavaThread::_cont_entry is non-null).  Available on all JDK 21+
+    // builds because _cont_entry is a field of JavaThread, which is always
+    // exported in gHotSpotVMStructs.  Does NOT require ContinuationEntry type
+    // size, so it works on JDK 21-26 where type_size() == 0.
+    bool isCarryingVirtualThread() const {
+        if (_cont_entry_offset < 0) return false;
+        return SafeAccess::loadPtr((void**) const_cast<VMThread*>(this)->at(_cont_entry_offset), nullptr) != nullptr;
+    }
+
+    // Returns the innermost active ContinuationEntry for this thread, or nullptr
+    // if none exists or ContinuationEntry layout is unavailable (JDK 21-26,
+    // where ContinuationEntry is not in gHotSpotVMTypes so type_size() == 0).
+    // Used by stackWalker to locate the enterSpecial frame when crossing the
+    // virtual-thread continuation boundary.
+    VMContinuationEntry* contEntry() {
+        if (_cont_entry_offset < 0 || VMContinuationEntry::type_size() == 0) return nullptr;
+        void* ptr = SafeAccess::loadPtr((void**) at(_cont_entry_offset), nullptr);
+        return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr;
+    }
+
     inline VMMethod* compiledMethod();
 private:
     static inline int nativeThreadId(JNIEnv* jni, jthread thread);

ddprof-lib/src/main/cpp/profiler.cpp

@@ -15,6 +15,7 @@
 #include "flightRecorder.h"
 #include "itimer.h"
 #include "hotspot/vmStructs.h"
+#include "hotspot/vmStructs.inline.h"
 #include "j9/j9Ext.h"
 #include "j9/j9WallClock.h"
 #include "jvmThread.h"
@@ -791,6 +792,24 @@ u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event
         // see https://github.com/async-profiler/async-profiler/pull/1090
         LP64_ONLY(frames[i].padding = 0;)
       }
+      // On JDK 21+, GetStackTrace on a virtual thread returns only the VT's
+      // logical stack; it stops at the continuation boundary and never includes
+      // carrier-thread frames.  Without a synthetic root the trace appears
+      // truncated to the UI backend, which attributes it to "Missing Frames".
+      // Detect the VT case via JavaThread::_cont_entry being non-null on the
+      // carrier.  This field is in gHotSpotVMStructs on all JDK 21+ builds so
+      // isCarryingVirtualThread() works regardless of JDK version.  Append a
+      // synthetic "JVM Continuation" root frame to mark the boundary
+      // explicitly, matching the behaviour of walkVM without carrier_frames.
+      if (VM::hotspot_version() >= 21 && num_frames < _max_stack_depth) {
+        VMThread* carrier = VMThread::current();
+        if (carrier != nullptr && carrier->isCarryingVirtualThread()) {
+          frames[num_frames].bci = BCI_NATIVE_FRAME;
+          frames[num_frames].method_id = (jmethodID) "JVM Continuation";
+          LP64_ONLY(frames[num_frames].padding = 0;)
+          num_frames++;
+        }
+      }
     }
 
     call_trace_id = _call_trace_storage.put(num_frames, frames, false, counter);
@@ -884,6 +903,17 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
             }
           }
           num_frames += java_frames;
+          // ASGCT stops at the continuation boundary for virtual threads (JDK 21+).
+          // Append a synthetic root frame so the UI does not show "Missing Frames".
+          if (java_frames > 0 && VM::hotspot_version() >= 21 && num_frames < _max_stack_depth) {
+            VMThread* carrier = VMThread::current();
+            if (carrier != nullptr && carrier->isCarryingVirtualThread()) {
+              frames[num_frames].bci = BCI_NATIVE_FRAME;
+              frames[num_frames].method_id = (jmethodID) "JVM Continuation";
+              LP64_ONLY(frames[num_frames].padding = 0;)
+              num_frames++;
+            }
+          }
         }
       }
     }