Qt Quick 3D - Stencil Outline Extension Example

// Copyright (C) 2023 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause

#include "outlinerenderextension.h"

#include <rhi/qrhi.h>

#include <QtQuick3D/qquick3dobject.h>
#include <ssg/qquick3dextensionhelpers.h>

#include <ssg/qssgrenderhelpers.h>
#include <ssg/qssgrenderextensions.h>
#include <ssg/qssgrendercontextcore.h>

class OutlineRenderer : public QSSGRenderExtension
{
public:
    OutlineRenderer() = default;

    bool prepareData(QSSGFrameData &data) override;
    void prepareRender(QSSGFrameData &data) override;
    void render(QSSGFrameData &data) override;
    void resetForFrame() override;
    RenderMode mode() const override { return RenderMode::Main; }
    RenderStage stage() const override { return RenderStage::PostColor; };

    QSSGPrepContextId stencilPrepContext { QSSGPrepContextId::Invalid };
    QSSGPrepContextId outlinePrepContext { QSSGPrepContextId::Invalid };
    QSSGPrepResultId stencilPrepResult { QSSGPrepResultId::Invalid };
    QSSGPrepResultId outlinePrepResult { QSSGPrepResultId::Invalid };
    QPointer<QQuick3DObject> model;
    QSSGNodeId modelId { QSSGNodeId::Invalid };
    QPointer<QQuick3DObject> material;
    QSSGResourceId outlineMaterialId {};
    float outlineScale = 1.05f;

    QSSGRenderablesId stencilRenderables;
    QSSGRenderablesId outlineRenderables;
};

bool OutlineRenderer::prepareData(QSSGFrameData &data)
{
    // Make sure we have a model and a material.
    if (!model || !material)
        return false;

    modelId = QQuick3DExtensionHelpers::getNodeId(*model);
    if (modelId == QSSGNodeId::Invalid)
        return false;

    outlineMaterialId = QQuick3DExtensionHelpers::getResourceId(*material);
    if (outlineMaterialId == QSSGResourceId::Invalid)
        return false;

    // This is the active camera for the scene (the camera used to render the QtQuick3D scene)
    QSSGCameraId camera = data.activeCamera();
    if (camera == QSSGCameraId::Invalid)
        return false;

    // We are going to render the same renderable(s) twice so we need to create two contexts.
    stencilPrepContext = QSSGRenderHelpers::prepareForRender(data, *this, camera, 0);
    outlinePrepContext = QSSGRenderHelpers::prepareForRender(data, *this, camera, 1);
    // Create the renderables for the target model. One for the original with stencil write, and one for the outline model.
    // Note that we 'Steal' the model here, that tells QtQuick3D that we'll take over the rendering of the model.
    stencilRenderables = QSSGRenderHelpers::createRenderables(data, stencilPrepContext, { modelId }, QSSGRenderHelpers::CreateFlag::Steal);
    outlineRenderables = QSSGRenderHelpers::createRenderables(data, outlinePrepContext, { modelId });

    // Now we can start setting the data for our models.
    // Here we set a material and a scale for the outline
    QSSGModelHelpers::setModelMaterials(data, outlineRenderables, modelId, { outlineMaterialId });
    QMatrix4x4 globalTransform = QSSGModelHelpers::getGlobalTransform(data, modelId);
    globalTransform.scale(outlineScale);
    QSSGModelHelpers::setGlobalTransform(data, outlineRenderables, modelId, globalTransform);

    // When all changes are done, we need to commit the changes.
    stencilPrepResult = QSSGRenderHelpers::commit(data, stencilPrepContext, stencilRenderables);
    outlinePrepResult = QSSGRenderHelpers::commit(data, outlinePrepContext, outlineRenderables);

    // If there's something to be rendered we return true.
    const bool dataReady = (stencilPrepResult != QSSGPrepResultId::Invalid && outlinePrepResult != QSSGPrepResultId::Invalid);

    return dataReady;
}

void OutlineRenderer::prepareRender(QSSGFrameData &data)
{
    Q_ASSERT(modelId != QSSGNodeId::Invalid);
    Q_ASSERT(stencilPrepResult != QSSGPrepResultId::Invalid && outlinePrepResult != QSSGPrepResultId::Invalid);

    const auto &ctx = data.contextInterface();

    if (const auto &rhiCtx = ctx->rhiContext()) {
        const QSSGRhiGraphicsPipelineState basePs = data.getPipelineState();
        QRhiRenderPassDescriptor *rpDesc = rhiCtx->mainRenderPassDescriptor();
        const int samples = rhiCtx->mainPassSampleCount();

        { // Original model - Write to the stencil buffer.
            QSSGRhiGraphicsPipelineState ps = basePs;
            ps.flags |= { QSSGRhiGraphicsPipelineState::Flag::BlendEnabled,
                          QSSGRhiGraphicsPipelineState::Flag::DepthWriteEnabled,
                          QSSGRhiGraphicsPipelineState::Flag::UsesStencilRef,
                          QSSGRhiGraphicsPipelineState::Flag::DepthTestEnabled };
            ps.stencilWriteMask = 0xff;
            ps.stencilRef = 1;
            ps.samples = samples;
            ps.cullMode = QRhiGraphicsPipeline::Back;

            ps.stencilOpFrontState = { QRhiGraphicsPipeline::Keep,
                                       QRhiGraphicsPipeline::Keep,
                                       QRhiGraphicsPipeline::Replace,
                                       QRhiGraphicsPipeline::Always };

            QSSGRenderHelpers::prepareRenderables(data, stencilPrepResult, rpDesc, ps);
        }

        { // Scaled version - Only draw outside the original.
            QSSGRhiGraphicsPipelineState ps = basePs;
            ps.flags |= { QSSGRhiGraphicsPipelineState::Flag::BlendEnabled,
                          QSSGRhiGraphicsPipelineState::Flag::UsesStencilRef,
                          QSSGRhiGraphicsPipelineState::Flag::DepthTestEnabled };
            ps.flags.setFlag(QSSGRhiGraphicsPipelineState::Flag::DepthWriteEnabled, false);
            ps.stencilWriteMask = 0;
            ps.stencilRef = 1;
            ps.cullMode = QRhiGraphicsPipeline::Back;

            ps.stencilOpFrontState = { QRhiGraphicsPipeline::Keep,
                                       QRhiGraphicsPipeline::Keep,
                                       QRhiGraphicsPipeline::Replace,
                                       QRhiGraphicsPipeline::NotEqual };

            QSSGRenderHelpers::prepareRenderables(data, outlinePrepResult, rpDesc, ps);
        }
    }
}

void OutlineRenderer::render(QSSGFrameData &data)
{
    Q_ASSERT(stencilPrepResult != QSSGPrepResultId::Invalid);

    const auto &ctx = data.contextInterface();
    if (const auto &rhiCtx = ctx->rhiContext()) {
        QRhiCommandBuffer *cb = rhiCtx->commandBuffer();
        cb->debugMarkBegin(QByteArrayLiteral("Stencil outline pass"));
        QSSGRenderHelpers::renderRenderables(data, stencilPrepResult);
        QSSGRenderHelpers::renderRenderables(data, outlinePrepResult);
        cb->debugMarkEnd();
    }
}

void OutlineRenderer::resetForFrame()
{
    stencilPrepContext = { QSSGPrepContextId::Invalid };
    stencilPrepResult = { QSSGPrepResultId::Invalid };
}

OutlineRenderExtension::~OutlineRenderExtension() {}

float OutlineRenderExtension::outlineScale() const
{
    return m_outlineScale;
}

void OutlineRenderExtension::setOutlineScale(float newOutlineScale)
{
    if (qFuzzyCompare(m_outlineScale, newOutlineScale))
        return;
    m_outlineScale = newOutlineScale;

    markDirty(Dirty::OutlineScale);

    emit outlineScaleChanged();
}

QQuick3DObject *OutlineRenderExtension::target() const
{
    return m_target;
}

void OutlineRenderExtension::setTarget(QQuick3DObject *newTarget)
{
    if (m_target == newTarget)
        return;
    m_target = newTarget;

    markDirty(Dirty::Target);

    emit targetChanged();
}

QSSGRenderGraphObject *OutlineRenderExtension::updateSpatialNode(QSSGRenderGraphObject *node)
{
    if (!node)
        node = new OutlineRenderer;

    OutlineRenderer *renderer = static_cast<OutlineRenderer *>(node);
    renderer->outlineScale = m_outlineScale;
    renderer->model = m_target;
    renderer->material = m_outlineMaterial;

    m_dirtyFlag = {};

    return node;
}

void OutlineRenderExtension::markDirty(Dirty v)
{
    m_dirtyFlag |= v;
    update();
}

QQuick3DObject *OutlineRenderExtension::outlineMaterial() const
{
    return m_outlineMaterial;
}

void OutlineRenderExtension::setOutlineMaterial(QQuick3DObject *newOutlineMaterial)
{
    if (m_outlineMaterial == newOutlineMaterial)
        return;

    m_outlineMaterial = newOutlineMaterial;

    markDirty(Dirty::OutlineMaterial);
    emit outlineMaterialChanged();
}