Render LOD into shadow maps

Fixed bug with last cascade not using proper transform.
Added LOD objects and terrain to shadow maps.
Removed depth based shadow fade for now since it
doesn't work properly with the new cascades.

src/NewVegas/nvse/Game.h

@@ -5587,7 +5587,7 @@ class BSShaderPPLightingProperty : public BSShaderLightingProperty {
 	};
 
 	UInt32					unk07C;
-	UInt32					unk080;
+	float					fMorphDistance;
 	NiColorAlpha			kHairTint;
 	NiColorAlpha			kLandBlendParams;
 	BSShaderTextureSet*		spTextureSet;
@@ -5625,4 +5625,17 @@ class SpeedTreeLeafShaderProperty : public SpeedTreeShaderLightingProperty {
 
 	LeafData* leafData;	// 088
 };
-assert(sizeof(SpeedTreeLeafShaderProperty) == 0x8C);
+assert(sizeof(SpeedTreeLeafShaderProperty) == 0x8C);
+
+class BGSTerrainManager {
+public:
+	static NiNode* GetRootLandLODNode() {
+		return *(NiNode**)0x11D86A8;
+	}
+	static NiNode* GetRootObjectLODNode() {
+		return *(NiNode**)0x11D8690;
+	}
+	static NiNode* GetRootTreeLODNode() {
+		return *(NiNode**)0x11D8690;
+	}
+};

src/NewVegas/nvse/GameNi.cpp

@@ -58,7 +58,9 @@ NiDX9ShaderDeclaration* NiDX9ShaderDeclaration::Create(NiDX9Renderer* apRenderer
 }
 
 float* const BSShaderManager::fDepthBias = (float*)0x1200458;
+float* const BSShaderManager::fLODLandDrop = (float*)0x11AD808;
 NiPoint3* const BSShaderManager::kCameraPos = (NiPoint3*)0x11F474C;
+NiPoint4* const BSShaderManager::kLoadedRange = (NiPoint4*)0x11F95F4;
 BSShader** BSShaderManager::pspShaders = (BSShader**)0x11F9548;
 
 ShadowSceneNode* BSShaderManager::GetShadowSceneNode(UInt32 aeType) {
@@ -367,3 +369,7 @@ bool NiBound::WithinFrustum(NiFrustumPlanes* arPlanes) {
 bool NiCamera::LookAtWorldPoint(const NiPoint3& kWorldPt, const NiPoint3& kWorldUp) {
 	return ThisCall<bool>(0xA701B0, this, &kWorldPt, &kWorldUp);
 }
+
+
+D3DXMATRIX* const ShadowLightShader::VertexConstants::WorldTranspose = (D3DXMATRIX*)0x11FECC8;
+NiPoint4* const ShadowLightShader::VertexConstants::LODLandParams = (NiPoint4*)0x11FA0B0;

src/NewVegas/nvse/GameNi.h

@@ -1948,6 +1948,23 @@ class ShadowLightShader : public BSShader{
 	NiD3DShaderConstantMap* spVertexConstantMap;
 	NiD3DShaderConstantMap* spPixelConstantMap2;
 	NiD3DShaderConstantMap* spVertexConstantMap2;
+
+	static __forceinline NiPoint4* GetConstant(int index) {
+		return &((NiPoint4*)0x11FA0C0)[index];
+	}
+
+	struct VertexConstants {
+		// 8
+		static D3DXMATRIX* const WorldTranspose;
+
+		// 12
+		static __forceinline NiPoint4* const GetHighDetailRange() {
+			return GetConstant(34);
+		}
+
+		// 19
+		static NiPoint4* const LODLandParams;
+	};
 };
 assert(sizeof(ShadowLightShader) == 0x8C);
 
@@ -2726,7 +2743,9 @@ class BSShaderManager {
 	};
 
 	static float* const fDepthBias;
+	static float* const fLODLandDrop;
 	static NiPoint3* const kCameraPos;
+	static NiPoint4* const kLoadedRange;
 
 	static BSShader** pspShaders;
 

src/core/RenderPass.cpp

@@ -306,4 +306,46 @@ void SpeedTreeShadowRenderPass::UpdateConstants(NiGeometry* Geo) {
 	RenderState->SetSamplerState(0, D3DSAMP_MAGFILTER, D3DTEXF_POINT, false);
 	RenderState->SetSamplerState(0, D3DSAMP_MINFILTER, D3DTEXF_POINT, false);
 	RenderState->SetSamplerState(0, D3DSAMP_MIPFILTER, D3DTEXF_POINT, false);
-}
+}
+
+
+TerrainLODPass::TerrainLODPass() {
+	PixelShader = TheShadowManager->ShadowMapPixel;
+	VertexShader = TheShadowManager->ShadowMapVertex;
+	RegisterConstants();
+}
+
+
+bool TerrainLODPass::AccumObject(NiGeometry* Geo) {
+	if (!Geo->geomData || !Geo->geomData->BuffData) return false; // discard objects without buffer data
+
+	BSShaderProperty* ShaderProperty = (BSShaderProperty*)Geo->GetProperty(NiProperty::PropertyType::kType_Shade);
+	if (!ShaderProperty || !ShaderProperty->IsLightingProperty()) return false;
+
+	GeometryList.push(Geo);
+	return true;
+}
+
+
+void TerrainLODPass::UpdateConstants(NiGeometry* Geo) {
+	ShadowsExteriorEffect::ShadowStruct* ShadowConstants = &TheShaderManager->Effects.ShadowsExteriors->Constants;
+	ShadowConstants->Data.x = 3.0f; // Type of geo (0 normal, 1 actors (skinned), 2 speedtree leaves, 3 terrain LOD)
+	ShadowConstants->Data.y = 0.0f; // Alpha Control
+	TheRenderManager->CreateD3DMatrix(&TheShaderManager->ShaderConst.ShadowWorld, &Geo->m_worldTransform);
+	D3DXMatrixTranspose(&Constants.WorldTranspose, &TheShaderManager->ShaderConst.ShadowWorld);
+
+	BSShaderPPLightingProperty* prop = (BSShaderPPLightingProperty*)Geo->GetProperty(NiProperty::PropertyType::kType_Shade);
+
+	Constants.LODLandParams.x = prop->fMorphDistance;
+	Constants.LODLandParams.y = *BSShaderManager::fLODLandDrop;
+
+	Constants.HighDetailRange.x = BSShaderManager::kLoadedRange->x - BSShaderManager::kCameraPos->x;
+	Constants.HighDetailRange.y = BSShaderManager::kLoadedRange->y - BSShaderManager::kCameraPos->y;
+	Constants.HighDetailRange.z = BSShaderManager::kLoadedRange->z - 15;
+	Constants.HighDetailRange.w = BSShaderManager::kLoadedRange->w - 15;
+
+	IDirect3DDevice9* Device = TheRenderManager->device;
+	Device->SetVertexShaderConstantF(140, (float*)&Constants.WorldTranspose, 4);
+	Device->SetVertexShaderConstantF(144, (float*)&Constants.HighDetailRange, 1);
+	Device->SetVertexShaderConstantF(145, (float*)&Constants.LODLandParams, 1);
+}

src/core/RenderPass.h

@@ -81,4 +81,20 @@ class SpeedTreeShadowRenderPass : public RenderPass {
 	bool AccumObject(NiGeometry* Geo);
 	void RegisterConstants(); 
 	void UpdateConstants(NiGeometry* Geo);
+};
+
+
+class TerrainLODPass : public RenderPass {
+public:
+	TerrainLODPass();
+
+	struct ConstantsStruct {
+		D3DXMATRIX	WorldTranspose;
+		D3DXVECTOR4 LODLandParams;
+		D3DXVECTOR4 HighDetailRange;
+	};
+	ConstantsStruct Constants;
+
+	bool AccumObject(NiGeometry* Geo);
+	void UpdateConstants(NiGeometry* Geo);
 };

src/core/ShadowManager.cpp

@@ -13,6 +13,7 @@ void ShadowManager::Initialize() {
 	TheShadowManager->alphaPass = new AlphaShadowRenderPass();
 	TheShadowManager->skinnedGeoPass = new SkinnedGeoShadowRenderPass();
 	TheShadowManager->speedTreePass = new SpeedTreeShadowRenderPass();
+	TheShadowManager->terrainLODPass = new TerrainLODPass();
 
 	// load the shaders
 	TheShadowManager->ShadowMapVertex = (ShaderRecordVertex*)ShaderRecord::LoadShader("ShadowMap.vso", "Shadows\\");
@@ -112,7 +113,7 @@ NiNode* ShadowManager::GetRefNode(TESObjectREFR* Ref, ShadowsExteriorEffect::For
 
 
 // Detect which pass the object must be added to
-void ShadowManager::AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms) {
+void ShadowManager::AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLODLand) {
 	auto timelog = TimeLogger();
 
 	NiGeometry* geo = static_cast<NiGeometry*>(NiObject);
@@ -124,6 +125,7 @@ void ShadowManager::AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVOb
 
 	if (skinnedGeoPass->AccumObject(geo)) {}
 	else if (speedTreePass->AccumObject(geo)) {}
+	else if (Forms->Lod && isLODLand && terrainLODPass->AccumObject(geo)) {}
 	else if (Forms->AlphaEnabled && alphaPass->AccumObject(geo)) {}
 	else geometryPass->AccumObject(geo);
 
@@ -132,7 +134,7 @@ void ShadowManager::AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVOb
 
 
 // go through the Object children and sort the ones that will be rendered based on their properties
-void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLand, NiFrustumPlanes *arPlanes) {
+void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLand, bool isLOD, NiFrustumPlanes *arPlanes) {
 	if (!NiObject) return;
 
 	std::stack<NiAVObject*> containers;
@@ -144,7 +146,7 @@ void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::F
 	if (!NiObject->IsGeometry())
 		containers.push(NiObject);
 	else
-		AccumObject(&containers, NiObject, Forms);
+		AccumObject(&containers, NiObject, Forms, isLand && isLOD);
 
 
 	// Gather geometry
@@ -168,7 +170,7 @@ void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::F
 			if (!child->IsGeometry())
 				containers.push(child);
 			else
-				AccumObject(&containers, child, Forms);
+				AccumObject(&containers, child, Forms, false);
 			continue;
 		}
 
@@ -178,7 +180,7 @@ void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::F
 			if (!isLand && child->GetWorldBoundRadius() < Forms->MinRadius) continue;
 
 			// Frustum culling.
-			if (arPlanes && isLand) {
+			if (arPlanes && (isLand || isLOD)) {
 				BSMultiBoundNode* multibound = child->IsMultiBoundNode();
 
 				if (multibound && !multibound->spMultiBound->spShape->WithinFrustum(*arPlanes)) continue;
@@ -189,7 +191,7 @@ void ShadowManager::AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::F
 			if (!child->IsGeometry())
 				containers.push(child);
 			else
-				AccumObject(&containers, child, Forms);
+				AccumObject(&containers, child, Forms, isLand && isLOD);
 		}
 	}
 }
@@ -203,6 +205,7 @@ void ShadowManager::RenderAccums(D3DVIEWPORT9* ViewPort) {
 	Device->BeginScene();
 
 	geometryPass->RenderAccum();
+	terrainLODPass->RenderAccum();
 	alphaPass->RenderAccum();
 	skinnedGeoPass->RenderAccum();
 	speedTreePass->RenderAccum();
@@ -225,6 +228,11 @@ void ShadowManager::RenderShadowMap(ShadowsExteriorEffect::ShadowMapSettings* Sh
 	RenderState->SetRenderState(D3DRS_DEPTHBIAS, (DWORD)0.0f, RenderStateArgs);
 	RenderState->SetRenderState(D3DRS_SLOPESCALEDEPTHBIAS, (DWORD)0.0f, RenderStateArgs);
 
+	if (ShadowMap->Forms.Lod) {
+		AccumChildren(BGSTerrainManager::GetRootLandLODNode(), &ShadowMap->Forms, true, true, &ShadowMap->ShadowMapFrustumPlanes);
+		AccumChildren(BGSTerrainManager::GetRootObjectLODNode(), &ShadowMap->Forms, false, true, &ShadowMap->ShadowMapFrustumPlanes);
+	}
+
 	if (Player->GetWorldSpace()) {
 		GridCellArray* CellArray = Tes->gridCellArray;
 		UInt32 CellArraySize = CellArray->size * CellArray->size;
@@ -246,10 +254,7 @@ void ShadowManager::AccumExteriorCell(TESObjectCELL* Cell, ShadowsExteriorEffect
 		return;
 
 	if (ShadowMap->Forms.Terrain)
-		AccumChildren(Cell->GetChildNode(TESObjectCELL::kCellNode_Land), &ShadowMap->Forms, true, &ShadowMap->ShadowMapFrustumPlanes);
-
-
-	// if (ShadowsExteriors->Forms[ShadowMapType].Lod) RenderLod(Tes->landLOD, ShadowMapType); //Render terrain LOD
+		AccumChildren(Cell->GetChildNode(TESObjectCELL::kCellNode_Land), &ShadowMap->Forms, true, false, &ShadowMap->ShadowMapFrustumPlanes);
 
 	TList<TESObjectREFR>::Entry* Entry = &Cell->objectList.First;
 	while (Entry) {
@@ -261,7 +266,7 @@ void ShadowManager::AccumExteriorCell(TESObjectCELL* Cell, ShadowsExteriorEffect
 		}
 
 		if (RefNode && RefNode->WithinFrustum(&ShadowMap->ShadowMapFrustumPlanes))
-			AccumChildren(RefNode, &ShadowMap->Forms, false, &ShadowMap->ShadowMapFrustumPlanes);
+			AccumChildren(RefNode, &ShadowMap->Forms, false, false, &ShadowMap->ShadowMapFrustumPlanes);
 
 		Entry = Entry->next;
 	}
@@ -323,7 +328,7 @@ void ShadowManager::RenderShadowSpotlight(NiSpotLight** Lights, UInt32 LightInde
 		D3DXVec3Normalize(&ObjectToLight, &ObjectToLight);
 		bool inFront = D3DXVec3Dot(&ObjectToLight, &CameraDirection) > 0;
 		if (inFront && RefNode->GetDistance(LightPos) <= Radius + RefNode->GetWorldBoundRadius()) 
-			AccumChildren(RefNode, &Settings->Forms, false);
+			AccumChildren(RefNode, &Settings->Forms, false, false);
 
 		Entry = Entry->next;
 	}
@@ -454,7 +459,7 @@ void ShadowManager::RenderShadowCubeMap(ShadowSceneLight** Lights, UInt32 LightI
 
 					D3DXVec3Normalize(&ObjectToLight, &ObjectToLight);
 					bool inFront = D3DXVec3Dot(&ObjectToLight, &CameraDirection) > 0;
-					if (RefNode->GetDistance(LightPos) <= Radius + RefNode->GetWorldBoundRadius()) AccumChildren(RefNode, &Settings->Forms, false);
+					if (RefNode->GetDistance(LightPos) <= Radius + RefNode->GetWorldBoundRadius()) AccumChildren(RefNode, &Settings->Forms, false, false);
 				}
 				Entry = Entry->next;
 			}
@@ -632,6 +637,8 @@ void ShadowManager::RenderShadowMaps() {
 		skinnedGeoPass->PixelShader = ShadowMapPixel;
 		speedTreePass->VertexShader = ShadowMapVertex;
 		speedTreePass->PixelShader = ShadowMapPixel;
+		terrainLODPass->VertexShader = ShadowMapVertex;
+		terrainLODPass->PixelShader = ShadowMapPixel;
 
 		if (ExteriorEnabled && SunDir.z > 0.0f) {
 			ShadowData->z = 0; // set shader constant to identify other shadow maps

src/core/ShadowManager.h

@@ -15,8 +15,8 @@ class ShadowManager { // Never disposed
 
 
 	NiNode*					GetRefNode(TESObjectREFR* Ref, ShadowsExteriorEffect::FormsStruct* Forms);
-	void					AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLand, NiFrustumPlanes* arPlanes = nullptr);
-	void					AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms);
+	void					AccumChildren(NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLand, bool isLOD, NiFrustumPlanes* arPlanes = nullptr);
+	void					AccumObject(std::stack<NiAVObject*>* containersAccum, NiAVObject* NiObject, ShadowsExteriorEffect::FormsStruct* Forms, bool isLODLand);
 	void					RenderAccums(D3DVIEWPORT9* Viewport);
 	D3DXMATRIX				GetViewMatrix(D3DXVECTOR3* At, D3DXVECTOR4* Dir);
 	void					RenderShadowMap(ShadowsExteriorEffect::ShadowMapSettings* ShadowMap, D3DXMATRIX* ViewProj);
@@ -30,6 +30,7 @@ class ShadowManager { // Never disposed
 	AlphaShadowRenderPass*			alphaPass;
 	SkinnedGeoShadowRenderPass*		skinnedGeoPass;
 	SpeedTreeShadowRenderPass*		speedTreePass;
+	TerrainLODPass*					terrainLODPass;
 
 	NiVector4				BillboardRight;
 	NiVector4				BillboardUp;

src/hlsl/NewVegas/Effects/SunShadows.fx.hlsl

@@ -96,8 +96,8 @@ float GetLightAmount(float4 coord, float depth)
     else if (length(coord.xyz - TESR_ShadowFarCenter.xyz) < TESR_ShadowFarCenter.w) {
         return GetLightAmountValue(TESR_ShadowCameraToLightTransformFar, coord, 0.0, 0.5, Bias.z, BleedReduction.z);
     }
-    else if (length(coord.xyz - TESR_ShadowFarCenter.xyz) < TESR_ShadowFarCenter.w) {
-        return GetLightAmountValue(TESR_ShadowCameraToLightTransformFar, coord, 0.5, 0.5, Bias.w, BleedReduction.w);
+    else if (length(coord.xyz - TESR_ShadowLodCenter.xyz) < TESR_ShadowLodCenter.w) {
+        return GetLightAmountValue(TESR_ShadowCameraToLightTransformLod, coord, 0.5, 0.5, Bias.w, BleedReduction.w);
     }
     else {
         return 1.0f;
@@ -181,7 +181,7 @@ float4 Shadow(VSOUT IN) : COLOR0
 
 	// Sample shadows from shadowmaps
 	float sunShadows = GetLightAmount(worldPos, depth); 
-	sunShadows = lerp(sunShadows, 1.0f, smoothstep(TESR_ShadowRadius.z, TESR_ShadowRadius.w, depth)); //fade shadows along last cascade
+	//sunShadows = lerp(sunShadows, 1.0f, smoothstep(TESR_ShadowRadius.z, TESR_ShadowRadius.w, depth)); //fade shadows along last cascade
 
 	Shadow.r = min(Shadow.r, sunShadows); // get the darkest between Screenspace & Sun shadows
 

src/hlsl/NewVegas/Shaders/Shadows/ShadowMap.vso.hlsl

@@ -11,9 +11,15 @@ float4 RustleParams : register(c66);
 float4 WindMatrices[16] : register(c67);
 float4 LeafBase[48] : register(c83);
 
+// Terrain LOD params for hiding it under normal terrain.
+row_major float4x4 WorldTranspose : register(c140);
+float4 HighDetailRange : register(c144);
+float4 LODLandParams : register(c145);
+
 struct VS_INPUT {
     float4 position : POSITION;
 	float4 texcoord_0 : TEXCOORD0;
+    float4 texcoord_1 : TEXCOORD1;  // Terrain LOD, no idea what it represents.
     float4 blendweight : BLENDWEIGHT;
     float4 blendindexes : BLENDINDICES;
 };
@@ -102,6 +108,15 @@ VS_OUTPUT main(VS_INPUT IN) {
 		q28.xyzw = mul(float4x4(WindMatrices[0 + offset.x].xyzw, WindMatrices[1 + offset.x].xyzw, WindMatrices[2 + offset.x].xyzw, WindMatrices[3 + offset.x].xyzw), q59.xyzw);
 		r0.xyzw = (IN.blendindexes.x * (q28.xyzw - q59.xyzw)) + q59.xyzw;
 	}
+    else if (TESR_ShadowData.x == 3.0f) { // Terrain LOD.
+        float4 r1 = IN.position;
+        r1.z = lerp(IN.texcoord_1.x, IN.position.z, LODLandParams.x);
+
+        float q0 = (abs(dot(WorldTranspose[1].xyzw, r1.xyzw) - HighDetailRange.y) < HighDetailRange.w ? 1.0 : 0.0);
+        float q1 = (abs(dot(WorldTranspose[0].xyzw, r1.xyzw) - HighDetailRange.x) < HighDetailRange.z ? 1.0 : 0.0);
+
+        r0.z = r1.z - ((q0.x * q1.x) * LODLandParams.y);
+    }
     if (TESR_ShadowData.x != 1.0f) r0 = mul(r0, TESR_ShadowWorldTransform);
 	r0 = mul(r0, TESR_ShadowViewProjTransform);