From 67e454b88bda2252a204309c06626f207bc42917 Mon Sep 17 00:00:00 2001
From: Nathan Gray <nathan.gray@shotover.com>
Date: Mon, 6 Jul 2026 17:12:02 -0600
Subject: [PATCH] Fix new-delete type mismatch in MSDK dispatcher

MFXInitEx allocates session handles as MFX_DISP_HANDLE_EX, but
MFXClose deletes them through a MFX_DISP_HANDLE pointer.  The base
class destructor is not virtual, so this is undefined behavior and
is reported by AddressSanitizer as new-delete-type-mismatch (size of
the allocated type: 1072 bytes; size of the deallocated type: 1048
bytes), aborting any ASan-instrumented process that opens and closes
a session through the dispatcher.

Making ~MFX_DISP_HANDLE virtual is not an option: the passthrough
functions cast the session pointer directly to struct _mfxSession*
and rely on the base subobject living at offset zero, and the header
documents that the handle layout must stay compatible.  Adding a
vtable pointer would shift the base subobject.

Instead, allocate MFX_DISP_HANDLE_EX at every handle creation site
(it only extends the base with POD members, so this is safe for the
two sites that previously allocated the base class) and spell the
matching type at every delete of a base-typed pointer.  Allocation
and deallocation types now agree on all paths, including handles
reaching MFXClose from MFXInitEx, MFXInitEx2, and MFXCloneSession.

Signed-off-by: Nathan Gray <nathan.gray@shotover.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 libvpl/src/windows/main.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libvpl/src/windows/main.cpp b/libvpl/src/windows/main.cpp
index 704d4ed7..5f7f65d1 100644
--- a/libvpl/src/windows/main.cpp
+++ b/libvpl/src/windows/main.cpp
@@ -557,7 +557,7 @@ mfxStatus MFXInitEx2(mfxVersion version,
         *session = 0;
 
         // allocate the dispatching handle and call-table
-        pHandle = new MFX_DISP_HANDLE(par.Version);
+        pHandle = new MFX_DISP_HANDLE_EX(par.Version);
     }
     catch (...) {
         return MFX_ERR_MEMORY_ALLOC;
@@ -578,7 +578,7 @@ mfxStatus MFXInitEx2(mfxVersion version,
         // unload the failed DLL
         if (MFX_ERR_NONE != mfxRes) {
             pHandle->Close();
-            delete pHandle;
+            delete (MFX_DISP_HANDLE_EX *)pHandle;
             return MFX_ERR_UNSUPPORTED;
         }
         else {
@@ -608,7 +608,7 @@ mfxStatus MFXClose(mfxSession session) {
             // can't unload library in that case.
             if (MFX_ERR_UNDEFINED_BEHAVIOR != mfxRes) {
                 // release the handle
-                delete pHandle;
+                delete (MFX_DISP_HANDLE_EX *)pHandle;
             }
         }
         catch (...) {
@@ -767,7 +767,7 @@ static mfxStatus AllocateCloneHandle(MFX_DISP_HANDLE *parentHandle, MFX_DISP_HAN
 
     try {
         // requested version matches original session
-        ch = new MFX_DISP_HANDLE(ph->apiVersion);
+        ch = new MFX_DISP_HANDLE_EX(ph->apiVersion);
     }
     catch (...) {
         return MFX_ERR_MEMORY_ALLOC;
@@ -794,7 +794,7 @@ static mfxStatus AllocateCloneHandle(MFX_DISP_HANDLE *parentHandle, MFX_DISP_HAN
     // unload the failed DLL
     if (sts) {
         ch->Close();
-        delete ch;
+        delete (MFX_DISP_HANDLE_EX *)ch;
         return MFX_ERR_UNSUPPORTED;
     }
     else {
@@ -859,7 +859,7 @@ mfxStatus MFXCloneSession(mfxSession session, mfxSession *clone) {
 
             if (mfxRes != MFX_ERR_NONE || cloneRT == NULL) {
                 // RT call failed, delete cloned session
-                delete cloneHandle;
+                delete (MFX_DISP_HANDLE_EX *)cloneHandle;
                 return MFX_ERR_UNSUPPORTED;
             }
             cloneHandle->session = cloneRT;