Testrun and documentation improvements

Makefile

@@ -4,15 +4,21 @@ PDFFILE=$(OUTDIR)/testrun.pdf
 TEXFILE_FULL=$(OUTDIR)/testrun_full.tex
 PDFFILE_FULL=$(OUTDIR)/testrun_full.pdf
 
-run: test_smoke pdf view pdf_full clean
+smoke: test_smoke pdf view pdf_full clean
 	@echo FINISHED...
 
-run_full: test_full pdf view pdf_full clean
+short: test_short pdf view pdf_full clean
+	@echo FINISHED...
+
+full: test_full pdf view pdf_full clean
 	@echo FINISHED...
 
 test_smoke:
 	venv/bin/python smart_brain_test.py test.all.smoke
 
+test_short:
+	venv/bin/python smart_brain_test.py test.all.short
+
 test_full:
 	venv/bin/python smart_brain_test.py test.all.full
 

smart_brain_test.py

@@ -5,6 +5,7 @@ from simulation.rooms import house
 import sys
 from tests.all import test_smarthome
 
+# TODO: Add testobject information (Info dict via mqtt) --> Report
 # TODO: Extend tests in simulation
 #         - Switching button functions (gfw_dirk, ffe.sleep)
 #         - Brightness button functions (gfw.dirk, ffe.sleep)

templates/run_statistic.tex

@@ -6,7 +6,7 @@
 	Number of possibly failed tests & \textcolor{% if testrun.number_of_possibly_failed_tests > 0%}{orange}{% else %}{black}{% endif %}{{ "{\\bf %d}" % testrun.number_of_possibly_failed_tests }}\\
 	Number of failed tests & \textcolor{% if testrun.number_of_failed_tests > 0%}{red}{% else %}{black}{% endif %}{{ "{\\bf %d}" % testrun.number_of_failed_tests }}\\
 	\midrule
-	Executionlevel    & {{ macros.latex_filter(testrun.testcase_names.get('%d' % testrun.testcase_execution_level, 'unknown')) }}\\
+	Executionlevel    & {{ macros.latex_filter(testrun.testcase_names.get(testrun.testcase_execution_level, 'unknown')) }}\\
 	Time consumption  & {{ '%.3fs' % testrun.time_consumption }}\\
 	\bottomrule
 \end{tabu}

tests/__init__.py

@@ -1,28 +1,39 @@
 import report
 import simulation.devices
 from unittest.test import equivalency_chk
+from unittest.jsonlog import TRUN_EXEC_LVL
 
 DT_TOGGLE = 0.3
 
 
 class test_collection(object):
+    TCEL_DICT = {
+        'full': report.TCEL_FULL,
+        'short': report.TCEL_SHORT,
+        'smoke': report.TCEL_SMOKE
+    }
+
     def __init__(self, test_instance):
         super().__init__()
         self.test_instance = test_instance
 
     def capabilities(self):
-        return ['full', 'smoke']
+        return ['full', 'short', 'smoke']
 
     def command(self, command):
         for member in self.test_instance.getmembers():
             obj = self.test_instance.getobjbyname(member)
             if id(obj) != id(self):
-                obj.test_all(report.TCEL_FULL if command == 'full' else report.TCEL_SMOKE)
+                tcel = self.TCEL_DICT.get(command, report.TCEL_FULL)
+                self.test_instance.tcl[TRUN_EXEC_LVL] = tcel
+                obj.test_all(tcel)
 
 
 class testcase(object):
-    def __init__(self):
+    METHOD_EXECUTED = {}
-        super().__init__()
+
+    def __init__(self, tcl):
+        self.tcl = tcl
         self.__test_list__ = []
 
     def capabilities(self):
@@ -36,8 +47,9 @@ class testcase(object):
             if tc_name != "test_all":
                 self.command(tc_name, tcel)
 
-    def command(self, command, tcel=report.TCEL_FULL):
+    def command(self, command, tcel=report.TCEL_SINGLE):
         simulation.devices.OUTPUT_ACTIVE = False
         tc = getattr(self, command)
+        self.tcl[TRUN_EXEC_LVL] = tcel
         tc(tcel)
         simulation.devices.OUTPUT_ACTIVE = True

tests/all.py

@@ -78,10 +78,11 @@ class test_smarthome(object):
         system_info[jsonlog.SYSI_SYSTEM] = platform.system()
         system_info[jsonlog.SYSI_KERNEL] = platform.release() + ' (%s)' % platform.version()
         system_info[jsonlog.SYSI_USERNAME] = getpass.getuser()
-        system_info[jsonlog.SYSI_PATH] = os.path.abspath(os.path.basename(__file__))
+        system_info[jsonlog.SYSI_PATH] = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
         self.tcl = report.testSession(['__unittest__', 'unittest', ROOT_LOGGER_NAME])
         self.tcl[jsonlog.MAIN_KEY_SYSTEM_INFO] = system_info
+        self.tcl["testcase_names"] = report.TCEL_NAMES
 
     def __eval_tcl__(self):
         path = os.path.abspath(os.path.join(os.path.basename(__file__), '..'))

tests/heating.py

@@ -1,4 +1,5 @@
 import config
+import inspect
 import report
 from tests import testcase, DT_TOGGLE
 import time
@@ -7,13 +8,17 @@ from unittest.test import equivalency_chk, greater_chk
 
 class testcase_heating(testcase):
     def __init__(self, tcl, videv, valve):
-        self.tcl = tcl
+        super().__init__(tcl)
         self.__videv__ = videv
         self.__valve__ = valve
         self.__default_temperature__ = config.DEFAULT_TEMPERATURE.get(self.__valve__.topic)
         self.__test_list__ = ["test_user_temperature_setpoint", "test_default_temperature", "test_summer_mode", "test_away_mode", "test_boost_mode"]
 
     def test_user_temperature_setpoint(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("User temperature setpoint test for device and virtual device: %s" %
                               self.__valve__.topic, tcel, self.__test_user_temperature_setpoint__, tcel)
 
@@ -38,6 +43,10 @@ class testcase_heating(testcase):
             equivalency_chk(self.__videv__.get(self.__videv__.KEY_VALVE_TEMPERATURE_SETPOINT), setp, tLogger, "Virtual device valve temperature")
 
     def test_default_temperature(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Default temperature test for device and virtual device: %s" %
                               self.__valve__.topic, tcel, self.__test_default_temperature__, tcel)
 
@@ -58,6 +67,10 @@ class testcase_heating(testcase):
         equivalency_chk(self.__valve__.get(self.__valve__.KEY_TEMPERATURE_SETPOINT), dtemp, tLogger, "Valve temperature setpoint")
 
     def test_summer_mode(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Summer mode test: %s" % self.__valve__.topic, tcel, self.__test_summer_mode__, tcel)
 
     def __test_summer_mode__(self, tLogger, tcel):
@@ -81,6 +94,10 @@ class testcase_heating(testcase):
         equivalency_chk(self.__valve__.get(self.__valve__.KEY_TEMPERATURE_SETPOINT), vtemp, tLogger, "Temperature setpoint")
 
     def test_away_mode(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Away mode test: %s" % self.__valve__.topic, tcel, self.__test_away_mode__, tcel)
 
     def __test_away_mode__(self, tLogger, tcel):
@@ -104,6 +121,10 @@ class testcase_heating(testcase):
         equivalency_chk(self.__valve__.get(self.__valve__.KEY_TEMPERATURE_SETPOINT), vtemp, tLogger, "Temperature setpoint")
 
     def test_boost_mode(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Boost mode test: %s" % self.__valve__.topic, tcel, self.__test_boost_mode__, tcel)
 
     def __test_boost_mode__(self, tLogger, tcel):

tests/light.py

@@ -1,3 +1,4 @@
+import inspect
 import report
 from tests import testcase, DT_TOGGLE
 import time
@@ -6,7 +7,7 @@ from unittest.test import equivalency_chk
 
 class testcase_light(testcase):
     def __init__(self, tcl, videv, sw_device, li_device):
-        self.tcl = tcl
+        super().__init__(tcl)
         self.videv = videv
         self.sw_device = sw_device
         self.li_device = li_device
@@ -19,6 +20,10 @@ class testcase_light(testcase):
             self.__test_list__.append('test_color_temp')
 
     def test_power_on_off(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Power On/ Off test for device and virtual device: %s" % self.sw_device.topic, tcel, self.__test_power_on_off__, tcel)
 
     def __test_power_on_off__(self, tLogger, tcel):
@@ -38,6 +43,10 @@ class testcase_light(testcase):
             equivalency_chk(self.sw_device.get(self.sw_device.KEY_OUTPUT_0), sw_state, tLogger, "Switching device state")
 
     def test_brightness(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Brightness test for device and virtual device: %s" % self.li_device.topic, tcel, self.__test_brightness__, tcel)
 
     def __test_brightness__(self, tLogger, tcel):
@@ -65,6 +74,10 @@ class testcase_light(testcase):
         tLogger.debug("Resetting precondition (Power off)")
 
     def test_color_temp(self, tcel=report.TCEL_FULL):
+        fname = inspect.currentframe().f_code.co_name
+        if tcel != report.TCEL_SMOKE or not self.METHOD_EXECUTED.get(fname, False):
+            self.METHOD_EXECUTED[fname] = True
+            #
             self.tcl.testCase("Color temperature test for device and virtual device: %s" % self.li_device.topic, tcel, self.__test_color_temp__, tcel)
 
     def __test_color_temp__(self, tLogger, tcel):

tests/synchronisation.py

@@ -6,8 +6,7 @@ from unittest.test import equivalency_chk
 
 class testcase_synchronisation(testcase):
     def __init__(self, tcl, sw_master, br_master, ct_master, *follower):
-        super().__init__()
+        super().__init__(tcl)
-        self.tcl = tcl
         self.sw_master = sw_master
         self.br_master = br_master
         self.ct_master = ct_master