(C++) Shader Preprocessor

BlindSide · Post by **BlindSide** » Fri Feb 29, 2008 4:51 am

CShaderPre.h:

#include <irrlicht.h>

class CShaderPreprocessor
{
public:
	CShaderPreprocessor(irr::video::IVideoDriver* driverIn);
	irr::core::stringc ppShader(irr::core::stringc shaderProgram);
	irr::core::stringc ppShaderFF(irr::core::stringc shaderProgram);
	void addShaderDefine(const irr::core::stringc name, const irr::core::stringc value = "");
	void CShaderPreprocessor::removeShaderDefine(const irr::core::stringc name);

private:
	void initDefineMap();

	irr::video::IVideoDriver* driver;
	irr::core::map<irr::core::stringc , irr::core::stringc> DefineMap;
};

CShaderPre.cpp:

Code: Select all

#include "CShaderPre.h"
#include <iostream>
#include <string>
#include <fstream>

using namespace irr;
using namespace video;
using namespace core;
using namespace scene;
using namespace io;

struct SDefineExp
{
	SDefineExp() : IfPos(-1), ElsePos(-1), EndPos(-1), IfExp(""), Inverse(false) {};
	s32 IfPos;
	s32 ElsePos;
	s32 EndPos;
	core::stringc IfExp;
	bool Inverse;
};

core::array<SDefineExp> grabDefineExpressions(core::stringc &shaderProgram)
{
	s32 CurrentSearchPos = 1;
	s32 FindHelper = 1;
	s32 FindHelper2 = 1;
	
	core::array<SDefineExp> DefineArray;

	// Dont bother stripping comments if theres no defines.
	if(CurrentSearchPos = shaderProgram.find("##ifdef") == -1)
		return DefineArray;

	// Strip all comments, they get in the way.
	while((CurrentSearchPos = shaderProgram.find("//")) > -1)
	{
		FindHelper = shaderProgram.findNext('\n',CurrentSearchPos);
		if(FindHelper != -1)
			for(u32 i = CurrentSearchPos;i < (u32)FindHelper;++i)
				shaderProgram[i] = ' ';
		else
			for(u32 i = CurrentSearchPos;i < shaderProgram.size();++i)
				shaderProgram[i] = ' ';
	}

	while((CurrentSearchPos = shaderProgram.find("/*")) > -1)
	{
		FindHelper = shaderProgram.find("*/");
		if(FindHelper > CurrentSearchPos)
			for(u32 i = CurrentSearchPos;i <= (u32)(FindHelper + 1);++i)
				shaderProgram[i] = ' ';
		else
			for(u32 i = CurrentSearchPos;i < shaderProgram.size();++i)
				shaderProgram[i] = ' ';
	}

	while((CurrentSearchPos = shaderProgram.find("##ifdef")) > -1)
	{
		SDefineExp DExp;
		
		DExp.IfPos = CurrentSearchPos;
		
		// Comment out the ##ifdef so that we do not find it again, and so that the compiler ignores it.
		shaderProgram[CurrentSearchPos] = '/';
		shaderProgram[CurrentSearchPos + 1] = '/';

		FindHelper = shaderProgram.findNext(' ',CurrentSearchPos);
		FindHelper2 = shaderProgram.findNext('\n',FindHelper);

		if(FindHelper == -1 || FindHelper2 == -1)
		{
			std::cerr << "Shader preprocessor encountered invalid if statement." << std::endl;
			return DefineArray;
		}

		// Find the appropriate expression and trim all white space.
		DExp.IfExp = shaderProgram.subString(FindHelper,FindHelper2 - FindHelper);
		DExp.IfExp.trim();
		
		// Record if its inverse and remove ! sign from expression.
		if(DExp.IfExp[0] == '!')
		{
			DExp.IfExp[0] = ' ';
			DExp.IfExp.trim();
			DExp.Inverse = true;
		}

		bool EndIfFound = false;

		FindHelper2 = CurrentSearchPos;
		s32 IfEndScope = 0;

		while(!EndIfFound)
		{
			FindHelper = shaderProgram.findNext('#',FindHelper2);

			if(FindHelper == -1 || FindHelper >= (s32)(shaderProgram.size() - 3))
			{
				std::cerr << "Shader preprocessor encountered unmatched if statement." << std::endl;
				return DefineArray;
			}

			if(IfEndScope < 0)
			{
				std::cerr << "Shader preprocessor encountered unmatched endif statement." << std::endl;
				return DefineArray;
			}
			
			if(shaderProgram[FindHelper + 1] != '#')
			{
				FindHelper2 = FindHelper + 1;
				continue;
			}
			else if(shaderProgram[FindHelper + 2] == 'i')
			{
				IfEndScope++;
			}
			else if(shaderProgram[FindHelper + 2] == 'e' && shaderProgram[FindHelper + 3] == 'n')
			{
				if(IfEndScope == 0)
					break;

				IfEndScope--;
			}
			else if(shaderProgram[FindHelper + 2] == 'e' && shaderProgram[FindHelper + 3] == 'l')
			{
				if(IfEndScope == 0)
				{
					if(DExp.ElsePos != -1)
					{
						std::cerr << "Shader preprocessor encountered duplicate else statements per if statement." << std::endl;
						return DefineArray;
					}

					// Comment out the ##else so that we do not find it again, and so that the compiler ignores it.
					shaderProgram[FindHelper] = '/';
					shaderProgram[FindHelper + 1] = '/';

					DExp.ElsePos = FindHelper;
				}
			}
			
			FindHelper2 = FindHelper + 2;
		}

		// Comment out the ##endif so that we do not find it again, and so that the compiler ignores it.
		shaderProgram[FindHelper] = '/';
		shaderProgram[FindHelper + 1] = '/';

		DExp.EndPos = FindHelper;
		
		// Add the define expression to the array.
		DefineArray.push_back(DExp);
	}

	return DefineArray;
}

CShaderPreprocessor::CShaderPreprocessor(irr::video::IVideoDriver* driverIn) : driver(driverIn) 
{initDefineMap();};

void CShaderPreprocessor::initDefineMap()
{
	if(driver->queryFeature(EVDF_TEXTURE_NPOT))
		DefineMap["EVDF_TEXTURE_NPOT"] = "";
	if(driver->queryFeature(EVDF_FRAMEBUFFER_OBJECT))
		DefineMap["EVDF_FRAMEBUFFER_OBJECT"] = "";
	if(driver->queryFeature(EVDF_VERTEX_SHADER_1_1))
		DefineMap["EVDF_VERTEX_SHADER_1_1"] = "";
	if(driver->queryFeature(EVDF_VERTEX_SHADER_2_0))
		DefineMap["EVDF_VERTEX_SHADER_2_0"] = "";
	if(driver->queryFeature(EVDF_VERTEX_SHADER_3_0))
		DefineMap["EVDF_VERTEX_SHADER_3_0"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_1_1))
		DefineMap["EVDF_PIXEL_SHADER_1_1"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_1_2))
		DefineMap["EVDF_PIXEL_SHADER_1_2"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_1_3))
		DefineMap["EVDF_PIXEL_SHADER_1_3"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_1_4))
		DefineMap["EVDF_PIXEL_SHADER_1_4"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_2_0))
		DefineMap["EVDF_PIXEL_SHADER_2_0"] = "";
	if(driver->queryFeature(EVDF_PIXEL_SHADER_3_0))
		DefineMap["EVDF_PIXEL_SHADER_3_0"] = "";

        // Comment this line out if it causes a compile error. (It requires the SVN version of Irrlicht.)
	DefineMap[driver->getVendorInfo()] = "";
}

void CShaderPreprocessor::addShaderDefine(const core::stringc name, const core::stringc value)
{
	// No need for this as its already inited at startup.
	//// If DefineMap is empty then initialize it.
	//if(DefineMap.isEmpty())
	//	initDefineMap();

	DefineMap[name] = value;
}

void CShaderPreprocessor::removeShaderDefine(const core::stringc name)
{
	DefineMap.remove(name);
}

//! PreProcesses a shader using Irrlicht's built-in shader preprocessor.
core::stringc CShaderPreprocessor::ppShader(core::stringc shaderProgram)
{
	core::array<SDefineExp> DefineArray = grabDefineExpressions(shaderProgram);

	// No need for this as its already inited at startup.
	//// If DefineMap is empty then initialize it.
	//if(DefineMap.isEmpty())
	//	initDefineMap();

	for(u32 i = 0; i < DefineArray.size();++i)
	{
		if(DefineArray[i].IfPos == -1)
			break;
	
		// Either it is true and not inversed, or it is false, but inversed. 
		// (Wish C++ had a built-in (logical) XOR operator sometimes. :P) 
		if((DefineMap.find(DefineArray[i].IfExp) && !DefineArray[i].Inverse) 
		|| (!DefineMap.find(DefineArray[i].IfExp) && DefineArray[i].Inverse))
		{
			if(DefineArray[i].ElsePos > -1)
			{
				// If there is an else statement then clear the else section.
				if(DefineArray[i].EndPos != -1)
				{
					for(int z = DefineArray[i].ElsePos;z <= DefineArray[i].EndPos + 6;++z)
						shaderProgram[z] = ' ';
				}
			}
		}
		else if(DefineArray[i].ElsePos != -1)
		{
			// If there is an else statement then clear the if section.
			for(int z = DefineArray[i].IfPos;z <= DefineArray[i].ElsePos + 5;++z)
				shaderProgram[z] = ' ';
		}
		else
		{
			// Else just clear the whole block.
			if(DefineArray[i].EndPos != -1)
			{
				for(int z = DefineArray[i].IfPos;z <= DefineArray[i].EndPos + 6;++z)
					shaderProgram[z] = ' ';
			}
		}
	}

	core::map<core::stringc,core::stringc>::ParentFirstIterator DefIter;
	s32 DefFinder = 1;

	// Replace all shader defines.
	for(DefIter = DefineMap.getParentFirstIterator();!DefIter.atEnd();DefIter++)
	{
		if(DefIter->getValue().size() == 0)
			continue;

		// Replace all occurances.
		while((DefFinder = shaderProgram.find(DefIter->getKey().c_str())) > -1)
		{
			// Clear the define from the code.
			for(u32 z = DefFinder;z < DefFinder + DefIter->getKey().size();++z)
				shaderProgram[z] = ' ';

			// Stitch value and shader program together. (Is there a faster way?)
			shaderProgram = shaderProgram.subString(0,DefFinder) + DefIter->getValue() 
					+ shaderProgram.subString(DefFinder,shaderProgram.size() - 1);
		}
	}

	return shaderProgram;
}

std::string getFileContent(const std::string pFile)
{
	std::ifstream File(pFile.c_str(), std::ios::in);
	std::string Content;

	if(File.is_open())
	{
		for(std::string Line; std::getline(File, Line);)
			Content += Line + "\n"; 

		File.close();
	}
	
	return Content;
}

//! PreProcesses a shader using the shader preprocessor.
core::stringc CShaderPreprocessor::ppShaderFF(core::stringc shaderProgram)
{
	return ppShader(getFileContent(shaderProgram.c_str()).c_str());
}

Usage:

1. In the shader:

Code: Select all

float xsat( float x) 
{
  return clamp( x, 0.0, 1.0);
}

uniform sampler2D ColoredTextureSampler;
uniform sampler2D RandMapSampler;

// This condition is evaluated at compile time.	
##ifdef 2_LAYER
uniform sampler2D DetailMap;
##endif

uniform sampler2D ShadowMapSampler;
varying float lightVal;

float testShadow(vec2 smTexCoord, vec2 offset, float realDistance, float darkness)
{
	float texDepth = texture2D( ShadowMapSampler, vec2( smTexCoord + offset)).g;

	return (texDepth <= realDistance) ? ( darkness / 2.0000 ) : ( 0.000000 );
}

vec2 offsetArray[16];

vec4 pixelMain( in vec4 SMPos, in vec4 MVar, in vec2 TexCoords, in vec2 TexCoords2)
{
	offsetArray[0] = vec2(0.0,0.0);
	offsetArray[1] = vec2(0.0,1.0 / float(float(MAPRES)));
	offsetArray[2] = vec2(1.0 / float(float(MAPRES)),1.0 / float(float(MAPRES)));
	offsetArray[3] = vec2(-1.0 / float(float(MAPRES)),-1.0 / float(MAPRES));
	offsetArray[4] = vec2(-2.0 / float(MAPRES),0.0);
	offsetArray[5] = vec2(0.0,-2.0 / float(MAPRES));
	offsetArray[6] = vec2(2.0 / float(MAPRES),-2.0 / float(MAPRES));
	offsetArray[7] = vec2(-2.0 / float(MAPRES),2.0 / float(MAPRES));
	offsetArray[8] = vec2(3.0 / float(MAPRES),0.0);
	offsetArray[9] = vec2(0.0,3.0 / float(MAPRES));
	offsetArray[10] = vec2(3.0 / float(MAPRES),3.0 / float(MAPRES));
	offsetArray[11] = vec2(-3.0 / float(MAPRES),-3.0 / float(MAPRES));
	offsetArray[12] = vec2(-4.0 / float(MAPRES),0.0);
	offsetArray[13] = vec2(0.0,-4.0 / float(MAPRES));
	offsetArray[14] = vec2(4.0 / float(MAPRES),-4.0 / float(MAPRES));
	offsetArray[15] = vec2(-4.0 / float(MAPRES),4.0 / float(MAPRES));


    SMPos.x  = (((SMPos.x  / SMPos.w ) / 2.00000) + 0.500000);
    SMPos.y  = (((SMPos.y  / SMPos.w ) / 2.00000) + 0.500000);
					
    if(xsat( SMPos.x  ) == SMPos.x && xsat( SMPos.y  ) == SMPos.y && SMPos.z > 0.0 && SMPos.z < MVar.z)
	{		
		vec2 shadCoord = vec2(SMPos.x,SMPos.y);
		
		vec4 randCol = texture2D( RandMapSampler, shadCoord * (MAPRES / 8));
		randCol /= 100.0;
				
		for(int i = 0;i < SAMPLE_AMOUNT ; i++) // SAMPLE_AMOUNT is a preprocessor define.
			MVar.y -= testShadow(shadCoord, offsetArray[i] + randCol.rg,MVar.x , MVar.w) / float(SAMPLE_AMOUNT);
	}
// This condition is evaluated at compile time.	
##ifdef 2_LAYER
    return (texture2D( ColoredTextureSampler, TexCoords) * texture2D( DetailMap, TexCoords2) * MVar.y);
##else
	return (texture2D( ColoredTextureSampler, TexCoords) * MVar.y);
##endif
}

// Main entry point.
void main() 
{
    vec4 finalCOl = pixelMain( vec4(gl_TexCoord[0]), vec4(gl_TexCoord[1]), vec2(gl_TexCoord[2]), vec2(gl_TexCoord[3]));

    gl_FragData[0] = finalCOl;
}

In the source:

Code: Select all

CShaderPreprocessor* sPP = new CShaderPreprocessor(driver);

sPP->addShaderDefine("SAMPLE_AMOUNT",1);
				Solid = gpu->addHighLevelShaderMaterial(
					sPP->ppShaderFF(SDFNV).c_str(),	"vertexMain", video::EVST_VS_2_0,
					sPP->ppShaderFF(SDFNP).c_str(), "pixelMain", video::EPST_PS_2_0,
					mc1, video::EMT_SOLID);

// Enable 2 texture layers.
				sPP->addShaderDefine("2_LAYER",1);

				sPP->addShaderDefine("SAMPLE_AMOUNT",1);
				Detail = gpu->addHighLevelShaderMaterial(
					sPP->ppShaderFF(SDFNV).c_str(),	"vertexMain", video::EVST_VS_2_0,
					sPP->ppShaderFF(SDFNP).c_str(), "pixelMain", video::EPST_PS_2_0,
					mc1, video::EMT_DETAIL_MAP);
 
				sPP->addShaderDefine("SAMPLE_AMOUNT",4);
				DetailPCF4 = gpu->addHighLevelShaderMaterial(
					sPP->ppShaderFF(SDFNV).c_str(),	"vertexMain", video::EVST_VS_2_0,
					sPP->ppShaderFF(SDFNP).c_str(), "pixelMain", video::EPST_PS_2_0,
					mc1, video::EMT_DETAIL_MAP);

				sPP->addShaderDefine("SAMPLE_AMOUNT",7);
				DetailPCF8 = gpu->addHighLevelShaderMaterial(
					sPP->ppShaderFF(SDFNV).c_str(),	"vertexMain", video::EVST_VS_2_0,
					sPP->ppShaderFF(SDFNP).c_str(), "pixelMain", video::EPST_PS_2_0,
					mc1, video::EMT_DETAIL_MAP);

delete sPP;

This thing has made my graphics programming life much easier, so use it liberally. There are many things you can do with this, the graphics card manufactorer is included as a define at the compile time of the shader, as are all the driver features automatically. So you can do this:

Code: Select all

##ifdef NVIDIA Corporation
// Do some NVIDIA specific stuff.
##else
// Do alternative for other card brands.
##endif

##ifdef EVDF_PIXEL_SHADER_2_0
// If PS 2.0 is supported perform appropriate technique.
##else
// Else do alternative for older shader versions.
##endif

// No need to do anything in c++ source for these automatic defines!

This can also be used for shader macro-ing, for example, in C++ source:

Code: Select all

irr::c8* DO_LIGHTING = "vec3 Normal = normalize(gl_NormalMatrix * gl_Normal);\n"
"vec3 ecPos = gl_ModelViewMatrix * gl_Vertex;\n"
"vec3 LightDir = vec3(gl_LightSource[i].position-ecPos);\n"
"vec3 NDotL = max(dot(Normal,normalize(LightDir)),0.0);\n"
"	\n"
"float Dist = length(LightDir);\n"
"NDotL *= 1.0 / (gl_LightSource[0].constantAttenuation + gl_LightSource\n"
"[0].linearAttenuation * Dist);\n"
"				\n"
"gl_FrontColor += gl_LightSource[i].diffuse * NDotL;\n"
 

sPP->addShaderDefine("DO_LIGHTING", DO_LIGHTING);

In shader:

Code: Select all

void main() 
{
        // This will automatically be replaced with the string from the C++ source.
	DO_LIGHTING

        // Some other possibilities:
        DO_SPECULAR
        DO_HWSKINNING
        etc
        
        // This makes it easy to mix'n'match shaders and re-use code!

	gl_Position = ftransform();
	gl_TexCoord[0] = gl_MultiTexCoord0;
	gl_TexCoord[1] = gl_MultiTexCoord1;	
}

Ok, so in reality its just an overhyped string parser. But the convenience of this thing really goes a long way.

PLEASE REMEMBER: This preprocessor uses 2 hashes, e.g. "##ifdef" instead of "#ifdef" to avoid collision with built-in shader preprocessors in GLSL and HLSL. The reason I didn't just use those preprocessors is so that I can define things from inside Irrlicht, and also this is more portable as it is the same for all shading languages, and can even be used in ASM shaders.

Cheers

FuzzYspo0N · Post by **FuzzYspo0N** » Fri Feb 29, 2008 6:19 am

Awsome work blindside, yet another great tool from you.

Im keen to check this out after work

fmx · Post by **fmx** » Fri Feb 29, 2008 7:09 am

thanks BlindSide!
this will be really useful... now we can build up a shader library for our projects, with minimal hardware conflict issues

arras · Post by **arras** » Fri Feb 29, 2008 8:27 am

Thanks BlindSide, so what it basically does is to build string out of pieces of code based on some input parameters ...right?
Problem is that since shaders are compiled at the beginning, I have to build large set of shader materials if I want to have rendered object react dynamically. Else I can end up doing lots of useless calculations most of the time.

BlindSide · Post by **BlindSide** » Fri Feb 29, 2008 8:46 am

Yeah thats pretty much the point. Theres nothing wrong with making a few material types, it wont really have a detrimental effect on performance or anything. All this does is save time and space on the actual shader code.

Its also good for dynamic configuration over a range of systems and hardware types.

Cheers

hybrid · Post by **hybrid** » Fri Feb 29, 2008 8:56 am

Oh, so you were still working on this one. Nice!

dlangdev · Post by **dlangdev** » Fri Feb 29, 2008 9:06 am

nice.