Commits

Riley Adams committed a912b09

Initial CTest setup (works in Linux only at the moment).

Comments (0)

Files changed (11)

 
 # Setup tests (before samples so that PlayPen is included in browser)
 if (OGRE_BUILD_TESTS)
+  # enable CTest
+  ENABLE_TESTING()
+  INCLUDE(CTest)
+  # add ctest config
+  #CONFIGURE_FILE(CTestCustom.cmake)
   add_subdirectory(Tests)
 endif ()
 

CTestConfig.cmake

+## This file should be placed in the root directory of your project.
+## Then modify the CMakeLists.txt file in the root directory of your
+## project to incorporate the testing dashboard.
+## # The following are required to uses Dart and the Cdash dashboard
+##   ENABLE_TESTING()
+##   INCLUDE(CTest)
+set(CTEST_PROJECT_NAME "OGRE3D")
+set(CTEST_NIGHTLY_START_TIME "00:00:00 EST")
+
+set(CTEST_DROP_METHOD "http")
+# this is temporary, my personal hosting:
+set(CTEST_DROP_SITE "rileyadams.net")
+set(CTEST_DROP_LOCATION "/CDash/submit.php?project=OGRE3D")
+set(CTEST_DROP_SITE_CDASH TRUE)

Tests/CMakeLists.txt

 
     endif (OIS_FOUND)
 
+# setup test scripting
+configure_file(CTestCustom.cmake ${CMAKE_BINARY_DIR}/CTestCustom.cmake)
+configure_file(PreTest.cmake ${CMAKE_BINARY_DIR}/Tests/PreTest.cmake)
+
 endif (OGRE_BUILD_TESTS)

Tests/CTestCustom.cmake

+set(CTEST_CUSTOM_PRE_TEST "cmake -P Tests/PreTest.cmake")

Tests/PreTest.cmake

+# Run prior to the CTest testing
+# This actually runs the Test Context
+
+# First remove any existing test results
+exec_program( "cmake -E chdir Tests rm TestResults*.txt")
+# Now run the Test Context for each render system
+exec_program( "cmake -E chdir Tests ../bin/TestContext")

Tests/VisualTests/CMakeLists.txt

 
 # Configure Visual Test build
 
+# Setup CTest
+#ENABLE_TESTING()
+
 # add Context directory
 add_subdirectory(Context)
 
 
 # add VTests plugin directory
 add_subdirectory(VTests)
+
+# now add our tests
+
+# Since CTest expects one command per test, but the Visual Testing setup runs all at once
+# (since unloading and reloading everything for each test wouldn't be practical) we simply
+# check an output file for passage of each test using CTest's regex checking
+
+# What we'll use to output the file (cat in Unix, type in Windows)
+set(TESTING_READ_FILE cat)
+
+# define a macro for these tests:
+macro (VisualTest test_name render_system)
+	add_test("${test_name}_${render_system}" ${TESTING_READ_FILE} 
+		"${CMAKE_BINARY_DIR}/Tests/TestResults_${render_system}.txt")
+	set_tests_properties("${test_name}_${render_system}" 
+		PROPERTIES FAIL_REGULAR_EXPRESSION 
+		"${test_name}=Failed")
+endmacro (VisualTest)
+
+# and the list of tests itself:
+VisualTest("PlayPen_16Textures" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_2Spotlights" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_AlphaToCoverage" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_AttachObjectsToBones" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BasicPlane" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BillboardAccurateFacing" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BillboardChain" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BillboardOrigins" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BillboardTextureCoords" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BlendDiffuseColour" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BlitSubTextures" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Bsp" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_BuildTangentOnAnimatedMesh" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_CameraSetDirection" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ClearScene" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_CompositorTechniqueSwitch" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_CompositorTextureShadows" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_CubeDDS" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_CustomProjectionMatrix" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_DepthBias" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_DepthShadowMap" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Distortion" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt1" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt1Alpha" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt1FromMemory" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt3" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt3FromMemory" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Dxt5" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_FarFromOrigin" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_GeometryShaders" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ImageCombine" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_InfiniteAAB" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_IntersectionSceneQuery" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_LiSPSM" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_LightClipPlanes" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_LightClipPlanesMoreLights" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_LightScissoring" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_LotsAndLotsOfEntities" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualBlending" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualBoneMovement" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualIlluminationStage" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualLOD" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualLOD_File" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObject2D" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectIndexed" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectIndexedUpdateLarger" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectIndexedUpdateSmaller" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectNonIndexed" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectNonIndexedUpdateLarger" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ManualObjectNonIndexedUpdateSmaller" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MaterialSchemes" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MaterialSchemesListener" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MaterialSchemesWithLOD" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MaterialSchemesWithMismatchedLOD" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MorphAnimNormals" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MorphAnimNoNormals" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MultiSceneManagersSimple" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_MultiViewports" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_NegativeScale" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_NormalMapMirroredUVs" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Ortho" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_PointSprites" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_PoseAnimNormals" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_PoseAnimNoNormals" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ProjectSphere" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_Projection" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_RaySceneQuery" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ReflectedBillboards" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ReinitialiseEntityAlteredMesh" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ReloadResources" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_RibbonTrail" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SRGBtexture" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SceneNodeTracking" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SerialisedColour" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ShadowLod" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SkeletalAnimation" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SkeletonAnimationOptimise" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SpotlightViewProj" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_StaticGeometry" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_StaticGeometryWithLOD" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_StencilGlow" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_StencilShadows" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_StencilShadowsMixedOpSubMeshes" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SubEntityVisibility" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_SuppressedShadows" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadows" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadowsCustomCasterMat" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadowsCustomReceiverMat" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadowsIntegrated" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadowsIntegratedPSSM" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TextureShadowsTransparentCaster" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_TransparencyMipMaps" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_VertexTexture" "OpenGLRenderingSubsystem")
+VisualTest("PlayPen_ViewportNoShadows" "OpenGLRenderingSubsystem")
+
+VisualTest("VTests_StencilShadows" "OpenGLRenderingSubsystem")
+VisualTest("VTests_Transparency" "OpenGLRenderingSubsystem")
+VisualTest("VTests_TextureEffects" "OpenGLRenderingSubsystem")
+VisualTest("VTests_Particles" "OpenGLRenderingSubsystem")
+VisualTest("VTests_CubeMapping" "OpenGLRenderingSubsystem")

Tests/VisualTests/Common/include/SimpleResultWriter.h

+
+/*
+-----------------------------------------------------------------------------
+This source file is part of OGRE
+    (Object-oriented Graphics Rendering Engine)
+For the latest info, see http://www.ogre3d.org/
+
+Copyright (c) 2000-2009 Torus Knot Software Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+-----------------------------------------------------------------------------
+*/
+
+#ifndef __SimpleResultWriter_H__
+#define __SimpleResultWriter_H__
+
+#include "Ogre.h"
+#include "TinyHTML.h"
+#include "ImageValidator.h"
+#include "TestBatch.h"
+#include "TestResultWriter.h"
+
+/** Writes a simple plaintext file with pass/fail result for each test */
+class SimpleResultWriter : public TestResultWriter
+{
+public:
+
+    SimpleResultWriter(const TestBatch& set1, const TestBatch& set2, ComparisonResultVectorPtr results)
+        :TestResultWriter(set1, set2, results){}
+
+protected:
+
+    virtual Ogre::String getOutput()
+	{
+		std::stringstream out;
+
+		for(int i = 0; i < mResults->size(); ++i)
+			out<<(*mResults)[i].testName<<"="<<((*mResults)[i].passed?"Passed":"Failed")<<"\n";
+
+		return out.str();
+	}
+
+};
+
+#endif

Tests/VisualTests/Context/CMakeLists.txt

 	../Common/include/ImageValidator.h
 	../Common/include/TestBatch.h
 	../Common/include/TestResultWriter.h
+	../Common/include/SimpleResultWriter.h
 	../Common/include/HTMLWriter.h
 	../Common/include/VisualTest.h
 	../Common/include/TinyHTML.h

Tests/VisualTests/Context/src/TestContext.cpp

 #include "SamplePlugin.h"
 #include "TestResultWriter.h"
 #include "HTMLWriter.h"
+#include "SimpleResultWriter.h"
 #include "OgreConfigFile.h"
 
 #include "OgrePlatform.h"
 
     // get the path and list of test plugins from the config file
     Ogre::ConfigFile testConfig;
-    testConfig.load("tests.cfg");
+    testConfig.load(mFSLayer->getConfigFilePath("tests.cfg"));
     mPluginDirectory = testConfig.getSetting("TestFolder");
 
     Ogre::ConfigFile::SectionIterator sections = testConfig.getSectionIterator();
 	else if(!success)
 		mRoot->setRenderSystem(0);
 
+	mRenderSystemName = mRoot->getRenderSystem() ? mRoot->getRenderSystem()->getName() : "";
+
 	return success;
 }
 //-----------------------------------------------------------------------
     if (mGenerateHtml && !mReferenceSet)
     {
         HtmlWriter* writer = 0;
+        SimpleResultWriter* simpleWriter = 0;
 
         // look for a reference set first
         Ogre::ConfigFile info;
                 if (mBatch->canCompareWith((*i)))
                 {
                     writer = OGRE_NEW HtmlWriter(*i, *mBatch, mBatch->compare((*i)));
+                    simpleWriter = OGRE_NEW SimpleResultWriter(*i, *mBatch, mBatch->compare((*i)));
                     break;
                 }
             }
         {
             ref = OGRE_NEW TestBatch(info, mOutputDir + mCompareWith);
             if (mBatch->canCompareWith(*ref))
+			{
                 writer = OGRE_NEW HtmlWriter(*ref, *mBatch, mBatch->compare(*ref));
+                simpleWriter = OGRE_NEW SimpleResultWriter(*ref, *mBatch, mBatch->compare(*ref));
+			}
         }
 
         if (writer)
             // plus a uniquely named one for this run
             writer->writeToFile(mOutputDir + "out.html");
             writer->writeToFile(mOutputDir + "TestResults_" + mBatch->name + ".html");
+
+			// also save a summary file for CTest to parse (in the bin directory for now, 
+			// since I'm unsure of how to get the path to the writable area from a CMake script)
+			Ogre::String rs;
+			for(int i = 0; i < mRenderSystemName.size(); ++i)
+				if(mRenderSystemName[i]!=' ')
+					rs += mRenderSystemName[i];
+            simpleWriter->writeToFile("TestResults_" + rs + ".txt");
+
             OGRE_DELETE writer;
+            OGRE_DELETE simpleWriter;
         }
 
 		OGRE_DELETE ref;

Tests/VisualTests/PlayPen/src/PlayPenBsp.cpp

 -----------------------------------------------------------------------------
 */
 #include "PlayPenTests.h"
+#include "FileSystemLayer.h"
 
 PlayPen_Bsp::PlayPen_Bsp()
 {
 	// Load Quake3 locations from a file
 	ConfigFile cf;
 	
-	cf.load("quakemap.cfg");
+	cf.load(mFSLayer->getConfigFilePath("quakemap.cfg"));
 	
 	String quakePk3 = cf.getSetting("Archive");
 	String quakeLevel = cf.getSetting("Map");

Tests/VisualTests/PlayPen/src/PlayPenTestPlugin.cpp

     addSample(new PlayPen_MorphAnimationWithoutNormals());
     addSample(new PlayPen_PoseAnimationWithNormals());
     addSample(new PlayPen_PoseAnimationWithoutNormals());
-
-	// new tests as of July 18
     addSample(new PlayPen_SceneNodeTracking());
     addSample(new PlayPen_StencilGlow());
     addSample(new PlayPen_TransparencyMipMaps());
     addSample(new PlayPen_AlphaToCoverage());
     addSample(new PlayPen_BlitSubTextures());
     addSample(new PlayPen_ImageCombine());
-
-	// NEW
     addSample(new PlayPen_WindowedViewportMode());
     addSample(new PlayPen_Bsp());
     addSample(new PlayPen_SkeletalAnimation());
     addSample(new PlayPen_TextureShadowsCustomCasterMat());
     addSample(new PlayPen_TextureShadowsCustomReceiverMat());
     addSample(new PlayPen_ManualObject2D());
-
-	// New June 26
     addSample(new PlayPen_LiSPSM());
     addSample(new PlayPen_MaterialSchemesWithLOD());
     addSample(new PlayPen_MaterialSchemesWithMismatchedLOD());
     addSample(new PlayPen_GeometryShaders());
     addSample(new PlayPen_VertexTexture());
 
-	// The below is a temporary summary of the conversion process
-	// I made a script that parsed the playpen and automatically set
-	// up the basics for all the tests, so now I'm working through
-	// one-by-one fixing specific issues, and eliminating some tests
-	// (namely those that aren't actually visual).
+	// Tests that could not be converted for various reasons
 
 	// INCOMPLETE?
 	//addSample(new PlayPen_Material());