Guide ACE (Anti-Cheat Expert) Screenshot Methods — DWM Shellcode & GPU Capture

[byte_corvus]

Anyone currently digging into how ACE (Anti-Cheat Expert) handles screen captures has probably realized they aren't relying on basic GDI or BitBlt grabs. This AC is notorious for its invasive behavior, and their screenshot implementation is no exception. ACE effectively injects shellcode directly into dwm.exe to facilitate captures using GPU-specific methods, falling back to DXGI duplication only when necessary.

You cant view this link please login.

Technical Breakdown

ACE utilizes two primary GPU-accelerated paths plus a robust fallback:

1. NVIDIA Path: Uses NvFBC64.dll. Note that ACE doesn't just call the API; they hook specific internal functions to bypass the hardware/license checks NVIDIA usually enforces on consumer cards. If you're trying to replicate this, you'll likely need a patched NvFBC or to emulate their hook logic.
2. AMD Path: Operates via the Advanced Media Framework (AMF). Unlike the NVIDIA route, the AMD implementation is reported to work mostly as-is using the standard AMF SDK factories.
3. Fallback: If GPU-specific methods fail, it drops back to the DXGI Desktop Duplication API.

Vendor Detection & Implementation

The core logic starts with identifying the hardware vendor via DXGI to decide which capture path to initialize. Here is a look at the vendor detection and the NVIDIA initialization structure used in these types of implementations:

#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include <dxgi1_2.h>
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <filesystem>
 
#pragma comment(lib, "dxgi.lib")
 
#ifdef HAVE_AMF_SDK
#include "amf-sdk/amf/public/include/core/Factory.h"
#include "amf-sdk/amf/public/include/core/Context.h"
#include "amf-sdk/amf/public/include/core/Surface.h"
#include "amf-sdk/amf/public/include/components/Component.h"
#include "amf-sdk/amf/public/include/components/DisplayCapture.h"
using namespace amf;
#endif
 
namespace fs = std::filesystem;
 
typedef unsigned int NvU32;
typedef int          NVFBCRESULT;
#define NVFBC_DLL_VERSION 0x70
#define NVFBC_TOSYS       0x1201u
 
typedef struct { NvU32 dwVersion; NvU32 dwInterfaceType; NvU32 dwMaxDisplayWidth; NvU32 dwMaxDisplayHeight; void* pDevice; void* pPrivateData; NvU32 dwPrivateDataSize; NvU32 dwInterfaceVersion; void* pNvFBC; NvU32 dwAdapterIdx; NvU32 dwNvFBCVersion; void* cudaCtx; void* pPrivateData2; NvU32 dwPrivateData2Size; NvU32 dwReserved[55]; void* pReserved[27]; } NVFBC_CREATE_PARAMS;
 
#define NVFBC_STRUCT_VERSION(T,v) ((NvU32)sizeof(T) | ((NvU32)(v)<<16) | ((NvU32)NVFBC_DLL_VERSION<<24))
#define NVFBC_CREATE_PARAMS_VER_2 NVFBC_STRUCT_VERSION(NVFBC_CREATE_PARAMS,2)
static_assert(sizeof(NVFBC_CREATE_PARAMS) == 512, "NVFBC_CREATE_PARAMS must be 512 bytes");
 
typedef struct { NvU32 dwWidth; NvU32 dwHeight; NvU32 dwBufferWidth; NvU32 dwReserved; NvU32 bOverlayActive; NvU32 bMustRecreate; NvU32 bFirstBuffer; NvU32 bHWMouseVisible; NvU32 bProtectedContent; NvU32 dwDriverInternalData; unsigned long long ulTimeStamp; NvU32 bStereoOn; NvU32 bIGPUCapture; NvU32 dwSourcePID; NvU32 dwReserved1[23]; void* pReserved2[8]; } NVFBC_FRAME_GRAB_INFO;
 
struct INvFBCToSys;
typedef NVFBCRESULT (__fastcall *pfn_SetUp_t)(INvFBCToSys*, NvU32, NvU32, void**, void**);
typedef NVFBCRESULT (__fastcall *pfn_GrabFrame_t)(INvFBCToSys*, NVFBC_FRAME_GRAB_INFO*, NvU32, NvU32, NvU32, NvU32, NvU32, NvU32, NvU32);
struct INvFBCToSys_Vtable { pfn_SetUp_t SetUp; pfn_GrabFrame_t GrabFrame; void* unused2; void* Release; };
struct INvFBCToSys { INvFBCToSys_Vtable* vt; };
 
typedef NVFBCRESULT (__cdecl *pfn_NvFBC_CreateEx_t)(void*);
typedef NVFBCRESULT (__cdecl *pfn_NvFBC_Enable_t)(NvU32);
 
static bool save_bmp_bottom_up(const wchar_t* path, int w, int h, const uint8_t* src, size_t pitch)
{
    HANDLE f = CreateFileW(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
    if (f == INVALID_HANDLE_VALUE)
        return false;
 
    BITMAPFILEHEADER fh{};
    BITMAPINFOHEADER ih{};
    fh.bfType        = 0x4D42;
    fh.bfOffBits     = sizeof(fh) + sizeof(ih);
    fh.bfSize        = fh.bfOffBits + (DWORD)w * h * 4;
    ih.biSize        = sizeof(ih);
    ih.biWidth       = w;
    ih.biHeight      = h;
    ih.biPlanes      = 1;
    ih.biBitCount    = 32;
    ih.biCompression = BI_RGB;
 
    DWORD n = 0;
    WriteFile(f, &fh, sizeof(fh), &n, nullptr);
    WriteFile(f, &ih, sizeof(ih), &n, nullptr);
    for (int y = h - 1; y >= 0; --y)
        WriteFile(f, src + (size_t)y * pitch, (DWORD)w * 4, &n, nullptr);
    CloseHandle(f);
    return true;
}
 
static bool save_bmp_top_down(const wchar_t* path, int w, int h, const uint8_t* src, size_t pitch)
{
    HANDLE f = CreateFileW(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
    if (f == INVALID_HANDLE_VALUE)
        return false;
 
    BITMAPFILEHEADER fh{};
    BITMAPINFOHEADER ih{};
    fh.bfType        = 0x4D42;
    fh.bfOffBits     = sizeof(fh) + sizeof(ih);
    fh.bfSize        = fh.bfOffBits + (DWORD)w * h * 4;
    ih.biSize        = sizeof(ih);
    ih.biWidth       = w;
    ih.biHeight      = -h;
    ih.biPlanes      = 1;
    ih.biBitCount    = 32;
    ih.biCompression = BI_RGB;
 
    DWORD n = 0;
    WriteFile(f, &fh, sizeof(fh), &n, nullptr);
    WriteFile(f, &ih, sizeof(ih), &n, nullptr);
    for (int y = 0; y < h; ++y)
        WriteFile(f, src + (size_t)y * pitch, (DWORD)w * 4, &n, nullptr);
    CloseHandle(f);
    return true;
}
 
static void build_path(wchar_t* out, size_t n, const wchar_t* tag)
{
    SYSTEMTIME st;
    GetLocalTime(&st);
    _snwprintf_s(out, n, _TRUNCATE,
        L"%s_%04u%02u%02u_%02u%02u%02u_%03u.bmp",
        tag,
        st.wYear, st.wMonth, st.wDay,
        st.wHour, st.wMinute, st.wSecond, st.wMilliseconds);
}
 
static bool capture_nvidia()
{
    HMODULE fbc = LoadLibraryW(L"NvFBC64.dll");
    if (!fbc)
         return false;
 
    auto Create = (pfn_NvFBC_CreateEx_t)GetProcAddress(fbc, "NvFBC_CreateEx");
    auto Enable = (pfn_NvFBC_Enable_t)  GetProcAddress(fbc, "NvFBC_Enable");
    if (!Create)
        return false;
 
    if (Enable)
        Enable(1);
 
    NVFBC_CREATE_PARAMS cp{};
    cp.dwVersion          = NVFBC_CREATE_PARAMS_VER_2;
    cp.dwInterfaceType    = NVFBC_TOSYS;
    cp.dwInterfaceVersion = 0x70020008u;
    cp.dwAdapterIdx       = 0;
 
    NVFBCRESULT cr = Create(&cp);
    if (cr != 0 || !cp.pNvFBC)
        return false;
 
    auto* obj = (INvFBCToSys*)cp.pNvFBC;
 
    uint8_t* buf  = nullptr;
    void*    diff = nullptr;
    NVFBCRESULT sr = obj->vt->SetUp(obj, 0 /*ARGB*/, 1, (void**)&buf, &diff);
    if (sr != 1 || !buf)
        return false;
 
    NVFBC_FRAME_GRAB_INFO gi{};
    NVFBCRESULT r = obj->vt->GrabFrame(obj, &gi, 0, 0 /*FULL*/, 0, 0, 0, 0, 0);
    if (r < 0)
        return false;
    if (gi.dwWidth == 0 || gi.dwHeight == 0)
        return false;
 
    wchar_t path[MAX_PATH];
    build_path(path, MAX_PATH, L"nvfbc");
    if (!save_bmp_bottom_up(path, (int)gi.dwWidth, (int)gi.dwHeight, buf, (size_t)gi.dwBufferWidth * 4))
        return false;
 
    wprintf(L"%s\n", path);
    return true;
}
 
#ifdef HAVE_AMF_SDK
static bool capture_amd()
{
    HMODULE hAmf = LoadLibraryW(AMF_DLL_NAME);
    if (!hAmf)
        return false;
 
    auto fnInit = (AMFInit_Fn)GetProcAddress(hAmf, AMF_INIT_FUNCTION_NAME);
    if (!fnInit)
    {
        FreeLibrary(hAmf);
        return false;
    }
 
    AMFFactory* fac = nullptr;
    if (fnInit(AMF_FULL_VERSION, &fac) != AMF_OK || !fac)
    {
        FreeLibrary(hAmf);
        return false;
    }
 
    AMFContext* ctx = nullptr;
    if (fac->CreateContext(&ctx) != AMF_OK)
    {
        FreeLibrary(hAmf);
        return false;
    }
    if (ctx->InitDX11(nullptr) != AMF_OK)
    {
        ctx->Release();
        FreeLibrary(hAmf);
        return false;
    }
 
    AMFComponent* cap = nullptr;
    if (fac->CreateComponent(ctx, AMFDisplayCapture, &cap) != AMF_OK)
    {
        ctx->Terminate();
        ctx->Release();
        FreeLibrary(hAmf);
        return false;
    }
 
    cap->SetProperty(AMF_DISPLAYCAPTURE_MONITOR_INDEX, (amf_int64)0);
    cap->SetProperty(AMF_DISPLAYCAPTURE_MODE, (amf_int64)AMF_DISPLAYCAPTURE_MODE_GET_CURRENT_SURFACE);
 
    if (cap->Init(AMF_SURFACE_BGRA, 0, 0) != AMF_OK)
    {
        cap->Release();
        ctx->Terminate();
        ctx->Release();
        FreeLibrary(hAmf);
        return false;
    }
 
    Sleep(200);
 
    AMFData* data = nullptr;
    for (int i = 0; i < 40; ++i)
    {
        if (cap->QueryOutput(&data) == AMF_OK && data)
            break;
        Sleep(50);
    }
    if (!data)
    {
        cap->Terminate();
        cap->Release();
        ctx->Terminate();
        ctx->Release();
        FreeLibrary(hAmf);
        return false;
    }
 
    AMFSurface* surf = nullptr;
    if (data->QueryInterface(AMFSurface::IID(), (void**)&surf) != AMF_OK || !surf)
    {
        data->Release();
        cap->Terminate();
        cap->Release();
        ctx->Terminate();
        ctx->Release();
        FreeLibrary(hAmf);
        return false;
    }
 
    surf->Convert(AMF_MEMORY_HOST);
    AMFPlane* plane = surf->GetPlane(AMF_PLANE_PACKED);
    bool ok = false;
    if (plane)
    {
        wchar_t path[MAX_PATH];
        build_path(path, MAX_PATH, L"amf");
        ok = save_bmp_top_down(
            path,
            plane->GetWidth(), plane->GetHeight(),
            (const uint8_t*)plane->GetNative(),
            (size_t)plane->GetHPitch());
        if (ok)
            wprintf(L"%s\n", path);
    }
 
    surf->Release();
    data->Release();
    cap->Terminate();
    cap->Release();
    ctx->Terminate();
    ctx->Release();
    FreeLibrary(hAmf);
    return ok;
}
#else
static bool capture_amd()
{
    return false;
}
#endif
 
static UINT detect_vendor_id()
{
    IDXGIFactory1* fac = nullptr;
    if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)&fac)))
        return 0;
 
    IDXGIAdapter1* adp = nullptr;
    UINT vid = 0;
    if (SUCCEEDED(fac->EnumAdapters1(0, &adp)) && adp)
    {
        DXGI_ADAPTER_DESC1 d{};
        adp->GetDesc1(&d);
        vid = d.VendorId;
        adp->Release();
    }
    fac->Release();
    return vid;
}
 
int wmain()
{
    UINT vid = detect_vendor_id();
    if (vid == 0x10DE)
        return capture_nvidia() ? 0 : 1;
    if (vid == 0x1002 || vid == 0x1022)
        return capture_amd() ? 0 : 1;
    return 2;
}

Spoiler: NVIDIA (NvFBC) vs AMD (AMF) Logic

For NVIDIA, the shellcode looks for NvFBC_CreateEx and NvFBC_Enable within the DLL. The capture is handled by grabbing frames to system memory (NVFBC_TOSYS).

On the AMD side, it initializes the AMF context, sets up a DisplayCapture component, and queries the output for a surface. The surface is then converted to AMF_MEMORY_HOST to extract the raw pixel data.

Bypass Considerations

If you're developing an internal or an overlay, you need to be aware that your frames are being scraped directly from the DWM composition. While many external cheats think they are "invisible" by using transparent windows, ACE's shellcode in DWM can see exactly what is being presented to the monitor, especially when they use these high-performance GPU capture methods.

While others are catching manual bans because they forgot DWM is compromised, Infocheats users should be looking at how to either strip the shellcode or intercept the capture calls within the DWM process space.

Who has successfully hooked the NvFBC dispatch table within dwm.exe to return clean frames?