From 5e4e737448ddcd347e09d5aff7e60caf2e7fdcf2 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 6 Feb 2017 13:40:42 -0800 Subject: [PATCH 01/42] Skip frameRestore test --- unittests/test_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 533608f2cd..fa7a719acf 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -49,7 +49,7 @@ def test_frameProperties(self): f.Close() - + @unittest.skip("Omitting test for now, requires full App.") def test_frameRestore(self): f = wx.Frame(self.frame, title="Title", pos=(50,50), size=(100,100)) f.Show() From 2c1c1473d1459413b1467fb42061405c1b9ab63b Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 6 Feb 2017 21:04:38 -0800 Subject: [PATCH 02/42] Bumpy groundwork for improving unittests with full App usability As mentioned, its pretty rough right now, but the goal iss to provide an features for making testing features that require a full running application quick and easy. --- unittests/atc.py | 159 ++++++++++++++++++++++++++++++++++++++++ unittests/test_atc.py | 45 ++++++++++++ unittests/test_frame.py | 37 +++++++++- 3 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 unittests/atc.py create mode 100644 unittests/test_atc.py diff --git a/unittests/atc.py b/unittests/atc.py new file mode 100644 index 0000000000..7e5c4ef55b --- /dev/null +++ b/unittests/atc.py @@ -0,0 +1,159 @@ +# Samuel Dunn +# Application Test Case +# allows testing wx features within the context of an application + +# Drawbacks: Current revision doesn't play into unittest's framework very well +# mimicking it more than anything +# Current revision only supports handling test events through a top level Frame +# derived from TestFrame, alternatives are being considered +# Only one TestCase class per module + +# Benefits: Allows unittesting features that suffer from only synthesizing an active event loop. + +# TODO: +# Create a decorator method to identify methods that are fully self sufficient tests, so TestDone +# can be called implicitly +# On The other hand, its only truly useful to know if the test completes as part of the method +# A decorator for describing methods that are a part of a test sequence, but not the entire sequence +# could be useful +# Explore TestWidget possibilty +# Explore commencing test schedule various ways (depending on OS) +# Change ApplicationTestCase to be a TestCase class generator, allowing plural testcase classes per module +# explore potential for a meta-class that decorates test related methods to ensure test failure on unhandled +# exception (related to first couple TODOs) + +__version__ = "0.0.1" + +import os +import sys +import unittest +import wx +import wx.lib.newevent + +TestEvent, EVT_TEST = wx.lib.newevent.NewEvent() + +TestEvent.caseiter = 0 + +class TestApplication(wx.App): + def OnInit(self): + f = ApplicationTestCase._frame() + f.Show(True) + return True + +""" +class TestWidget: + # provides some useful methods for use within test widgets. + # it is recommended to derive test objects from this class in addtion to the + # relevate wx class + + # provide methods for: ensuring application closes (with and without an error code) + # standardized assertion + # etc. + + # should this class provide methods for event handling (forcing derivation from wx.EventHandler)? +""" + +class TestFrame(wx.Frame): + def __init__(self): + wx.Frame.__init__(self, None, wx.NewId(), "Phoenix Application Test") + self.__timer = wx.Timer(self, wx.NewId()) + + self.Bind(wx.EVT_TIMER, self.OnCommence, self.__timer) + self.Bind(EVT_TEST, self.OnTest) + + self.__timer.StartOnce(500) # Wait for the application mainloop to start and commence testing. + # while 500 milliseconds should be more than plenty, this is still + # clumsy and I don't like it. + # Initially I had an initial TestEvent posted to be processed when + # the mainloop was running, but this caused issues on some linux + # distributions. + # Ultimately I'd like to minimize as much waiting as feasibly possible + + def OnCommence(self, evt): + assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding + # commencing with the timer. + + # ensure the test schedule is set up, create it if it does not exist + if not hasattr(self, "schedule"): + self.schedule = [member[5:] for member in dir(self) if member.startswith("test_")] + + self.__results = [] + + # start test sequence + evt = TestEvent() + wx.PostEvent(self, evt) + + def OnTest(self, evt): + # find test case and try it + try: + testname = self.schedule[TestEvent.caseiter] + TestEvent.caseiter += 1 + except IndexError: + # well either some one made schedule some funky object + # or the end of the schedule has been reached. + # for now pretend its a good thing: + print("Reached end of schedule") + self.WrapUp() + return + + try: + testfunc = getattr(self, "test_%s" % testname) + except AttributeError: + print("Missing test method for: %s" % testname, file = sys.stderr) + sys.exit(1) + + try: + print("Testing: %s" % testname) + testfunc() + except Exception as e: + import traceback + _, _, tb = sys.exc_info() + traceback.print_tb(tb) + tb_info = traceback.extract_tb(tb) + + filename, line, func, text = tb_info[-1] + print("Additional Info:\nFile: %s\nLine: %s\nFunc: %s\nText: %s" % (filename, line, func, text), file = sys.stderr) + sys.exit(1) + + def TestDone(self, passed = True): + # record test status and invoke next test + if passed: + self.__results.append(".") + else: + self.__results.append("F") + + evt = TestEvent() + wx.PostEvent(self, evt) + + + def Assert(self, expr, message = "An Assertion occurred"): + # well, misnomer. Is a psuedo assert + # use this method when ensuring the test can continue + # in most test-failure scenarios you should use TestDone(False) + if not expr: + print(message, file = sys.stderr) + sys.exit(1) + + def WrapUp(self): + print("Results: %s" % "".join(self.__results)) + + if "F" in self.__results: + print("Test(s) have failed", file = sys.stderr) + sys.exit(1) + else: + # close peacefully + self.Close() + +class ApplicationTestCase(unittest.TestCase): + _app = TestApplication + _frame = TestFrame + + def test_application(self): + app = self._app() + app.MainLoop() + +def main(): + unittest.main() + +if __name__ == "__main__": + main() diff --git a/unittests/test_atc.py b/unittests/test_atc.py new file mode 100644 index 0000000000..7ecf4b4469 --- /dev/null +++ b/unittests/test_atc.py @@ -0,0 +1,45 @@ +from unittests import atc +import wx +import unittest + +# the testception is strong with this one. +class FrameRestoreTester(atc.TestFrame): + def __init__(self): + atc.TestFrame.__init__(self) + self.SetLabel("Frame Restore Test") + + # enforce a strict schedule + self.schedule = ("iconize", "restore", "maximize", "restore") + + def test_iconize(self): + self.Iconize() + wx.CallLater(250, self.Ensure, "Iconized") + + def test_maximize(self): + self.Maximize() + wx.CallLater(250, self.Ensure, "Maximized") + + def test_restore(self): + self.Restore() + wx.CallLater(250, self.Ensure, "Restored") + + def Ensure(self, ensurable): + if ensurable == "Iconized": + self.Assert(self.IsIconized()) + elif ensurable == "Maximized": + self.Assert(self.IsMaximized()) + elif ensurable == "Restored": + self.Assert(not self.IsIconized()) + self.Assert(not self.IsMaximized()) + + self.TestDone() + +# unittest only finds classes at local module scope that derive +# from TestCase, ways around this should be investigated +tc = atc.ApplicationTestCase + +# monkey wrench desired TestFrame class into the test case +tc._frame = FrameRestoreTester + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/unittests/test_frame.py b/unittests/test_frame.py index fa7a719acf..af2b015dd7 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -1,8 +1,8 @@ import unittest -from unittests import wtc +from unittests import wtc, atc import wx import os - +import sys #--------------------------------------------------------------------------- class frame_Tests(wtc.WidgetTestCase): @@ -60,7 +60,40 @@ def test_frameRestore(self): self.myYield() assert not f.IsMaximized() f.Close() +#--------------------------------------------------------------------------- + +# this is the frame that is used for testing Restore functionality +class FrameRestoreTester(atc.TestFrame): + def __init__(self): + atc.TestFrame.__init__(self) + self.SetLabel("Frame Restore Test") + + # enforce a strict schedule + self.schedule = ("iconize", "restore", "maximize", "restore") + + def test_iconize(self): + self.Iconize() + wx.CallLater(250, self.Ensure, "Iconized") + + def test_maximize(self): + self.Maximize() + wx.CallLater(250, self.Ensure, "Maximized") + + def test_restore(self): + self.Restore() + wx.CallLater(250, self.Ensure, "Restored") + + def Ensure(self, ensurable): + if ensurable == "Iconized": + self.TestDone(self.IsIconized()) + elif ensurable == "Maximized": + self.TestDone(self.IsMaximized()) + elif ensurable == "Restored": + self.TestDone(not self.IsIconized() and not self.IsMaximized()) + +tc = atc.ApplicationTestCase +tc._frame = FrameRestoreTester #--------------------------------------------------------------------------- From 5dbed799f4643c20704d03b84fd7f16da44c8a7b Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Feb 2017 23:25:35 +0000 Subject: [PATCH 03/42] now generating app/frame/testcase classes --- unittests/atc.py | 84 ++++++++++++++++++++++++++++++----------- unittests/test_frame.py | 10 ++--- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 7e5c4ef55b..e129dadf47 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -11,7 +11,7 @@ # Benefits: Allows unittesting features that suffer from only synthesizing an active event loop. # TODO: -# Create a decorator method to identify methods that are fully self sufficient tests, so TestDone +# Create a decorator method to identify methods that are fully self sufficient tests, so TestDone # can be called implicitly # On The other hand, its only truly useful to know if the test completes as part of the method # A decorator for describing methods that are a part of a test sequence, but not the entire sequence @@ -34,15 +34,9 @@ TestEvent.caseiter = 0 -class TestApplication(wx.App): - def OnInit(self): - f = ApplicationTestCase._frame() - f.Show(True) - return True - """ class TestWidget: - # provides some useful methods for use within test widgets. + # provides some useful methods for use within test widgets. # it is recommended to derive test objects from this class in addtion to the # relevate wx class @@ -53,9 +47,10 @@ class TestWidget: # should this class provide methods for event handling (forcing derivation from wx.EventHandler)? """ -class TestFrame(wx.Frame): +class TestWidget: def __init__(self): - wx.Frame.__init__(self, None, wx.NewId(), "Phoenix Application Test") + assert isinstance(self, wx.EvtHandler), "Test widget needs to be an event handler" + self.__timer = wx.Timer(self, wx.NewId()) self.Bind(wx.EVT_TIMER, self.OnCommence, self.__timer) @@ -63,9 +58,9 @@ def __init__(self): self.__timer.StartOnce(500) # Wait for the application mainloop to start and commence testing. # while 500 milliseconds should be more than plenty, this is still - # clumsy and I don't like it. - # Initially I had an initial TestEvent posted to be processed when - # the mainloop was running, but this caused issues on some linux + # clumsy and I don't like it. + # Initially I had an initial TestEvent posted to be processed when + # the mainloop was running, but this caused issues on some linux # distributions. # Ultimately I'd like to minimize as much waiting as feasibly possible @@ -132,25 +127,72 @@ def Assert(self, expr, message = "An Assertion occurred"): # in most test-failure scenarios you should use TestDone(False) if not expr: print(message, file = sys.stderr) - sys.exit(1) + raise Exception("Test Failure") def WrapUp(self): print("Results: %s" % "".join(self.__results)) - + if "F" in self.__results: print("Test(s) have failed", file = sys.stderr) sys.exit(1) + else: # close peacefully self.Close() -class ApplicationTestCase(unittest.TestCase): - _app = TestApplication - _frame = TestFrame - def test_application(self): - app = self._app() - app.MainLoop() + +class TestFrame(wx.Frame, TestWidget): + def __init__(self): + wx.Frame.__init__(self, None, wx.NewId(), "Phoenix Application Test") + TestWidget.__init__(self) + + +def CreateApp(frame): + class TestApp(wx.App): + def OnInit(self): + f = frame() + f.Show(True) + return True + return TestApp + +def CreateFrame(widget): + # called when the test widget is not a frame. + class BaseTestFrame(wx.Frame): + def __init__(self): + wx.Frame.__init__(self, None, wx.NewId(), "%s Test Frame" % str(type(widget))) + + sizer = wx.BoxSizer() + self.widget = widget(self) # assumes need of parent. + sizer.add(self.widget, 1, wx.EXPAND) + + self.SetSizer(sizer) + sizer.Layout() + + +def CreateATC(widget = TestFrame): + # if widget is not instance of TestFrame, generate a quick frame + # to house the widget + assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" + + tlw = None + if not issubclass(widget, wx.Frame): + # need to stick this widget in a frame + tlw = CreateFrame(widget) + + else: + tlw = widget + + app = CreateApp(tlw) + app.targetwidget = widget + + class ApplicationTestCase(unittest.TestCase): + # generate test case wrappers? + def test_application(self): + a = app() + a.MainLoop() + + return ApplicationTestCase def main(): unittest.main() diff --git a/unittests/test_frame.py b/unittests/test_frame.py index af2b015dd7..58dffd5183 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -63,9 +63,10 @@ def test_frameRestore(self): #--------------------------------------------------------------------------- # this is the frame that is used for testing Restore functionality -class FrameRestoreTester(atc.TestFrame): +class FrameRestoreTester(wx.Frame, atc.TestWidget): def __init__(self): - atc.TestFrame.__init__(self) + wx.Frame.__init__(self, None, wx.NewId(), "Frame Rstore Test") + atc.TestWidget.__init__(self) self.SetLabel("Frame Restore Test") # enforce a strict schedule @@ -90,10 +91,9 @@ def Ensure(self, ensurable): self.TestDone(self.IsMaximized()) elif ensurable == "Restored": self.TestDone(not self.IsIconized() and not self.IsMaximized()) - -tc = atc.ApplicationTestCase -tc._frame = FrameRestoreTester + +tc = atc.CreateATC(widget = FrameRestoreTester) #--------------------------------------------------------------------------- From 66416d9214f5e77b3f6c27226c99ab9b8003483d Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Feb 2017 23:26:18 +0000 Subject: [PATCH 04/42] Removed now redundant TestFrame class from ATC --- unittests/atc.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index e129dadf47..13f1665d8c 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -140,14 +140,6 @@ def WrapUp(self): # close peacefully self.Close() - - -class TestFrame(wx.Frame, TestWidget): - def __init__(self): - wx.Frame.__init__(self, None, wx.NewId(), "Phoenix Application Test") - TestWidget.__init__(self) - - def CreateApp(frame): class TestApp(wx.App): def OnInit(self): From 73106e2a68588f2cccda05638206c09b03ffc7e0 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Feb 2017 23:49:58 +0000 Subject: [PATCH 05/42] minor errors --- unittests/atc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 13f1665d8c..7561b0ebc4 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -78,6 +78,10 @@ def OnCommence(self, evt): evt = TestEvent() wx.PostEvent(self, evt) + def OnWatchDog(self, evt): + # this handler is not used yet. + assert 0, "Watchdog failure." + def OnTest(self, evt): # find test case and try it try: @@ -156,13 +160,14 @@ def __init__(self): sizer = wx.BoxSizer() self.widget = widget(self) # assumes need of parent. - sizer.add(self.widget, 1, wx.EXPAND) + sizer.Add(self.widget, 1, wx.EXPAND) self.SetSizer(sizer) sizer.Layout() + return BaseTestFrame -def CreateATC(widget = TestFrame): +def CreateATC(widget): # if widget is not instance of TestFrame, generate a quick frame # to house the widget assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" From 4e1dc22652cc0a4a78ba616e6737217d112f569f Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Feb 2017 19:22:49 -0800 Subject: [PATCH 06/42] Ensure test closes regardless of target widget --- unittests/atc.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 7561b0ebc4..483f2ad12e 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -142,7 +142,8 @@ def WrapUp(self): else: # close peacefully - self.Close() + for window in wx.GetTopLevelWindows(): + window.Close() def CreateApp(frame): class TestApp(wx.App): @@ -191,8 +192,5 @@ def test_application(self): return ApplicationTestCase -def main(): - unittest.main() - if __name__ == "__main__": main() From 7aefc60ea0f2f5ec379217418f631b33ea2a68be Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Feb 2017 19:23:23 -0800 Subject: [PATCH 07/42] Update a couple test files to ensure particular features are working --- unittests/test_atc.py | 44 ++++++++++------------------------------- unittests/test_frame.py | 2 +- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 7ecf4b4469..9e1ddafce0 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -1,45 +1,21 @@ from unittests import atc import wx import unittest +import random -# the testception is strong with this one. -class FrameRestoreTester(atc.TestFrame): - def __init__(self): - atc.TestFrame.__init__(self) - self.SetLabel("Frame Restore Test") +random.seed() - # enforce a strict schedule - self.schedule = ("iconize", "restore", "maximize", "restore") +class PanelColorChangeTester(wx.Panel, atc.TestWidget): + def __init__(self, parent): + wx.Panel.__init__(self, parent, wx.NewId()) + atc.TestWidget.__init__(self) - def test_iconize(self): - self.Iconize() - wx.CallLater(250, self.Ensure, "Iconized") + self.SetBackgroundColour(wx.BLUE) - def test_maximize(self): - self.Maximize() - wx.CallLater(250, self.Ensure, "Maximized") + def test_dummy(self): + self.TestDone(True) - def test_restore(self): - self.Restore() - wx.CallLater(250, self.Ensure, "Restored") - - def Ensure(self, ensurable): - if ensurable == "Iconized": - self.Assert(self.IsIconized()) - elif ensurable == "Maximized": - self.Assert(self.IsMaximized()) - elif ensurable == "Restored": - self.Assert(not self.IsIconized()) - self.Assert(not self.IsMaximized()) - - self.TestDone() - -# unittest only finds classes at local module scope that derive -# from TestCase, ways around this should be investigated -tc = atc.ApplicationTestCase - -# monkey wrench desired TestFrame class into the test case -tc._frame = FrameRestoreTester +testcase = atc.CreateATC(PanelColorChangeTester) if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 58dffd5183..f1b7446a99 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -65,7 +65,7 @@ def test_frameRestore(self): # this is the frame that is used for testing Restore functionality class FrameRestoreTester(wx.Frame, atc.TestWidget): def __init__(self): - wx.Frame.__init__(self, None, wx.NewId(), "Frame Rstore Test") + wx.Frame.__init__(self, None, wx.NewId(), "Frame Restore Test") atc.TestWidget.__init__(self) self.SetLabel("Frame Restore Test") From 3cfc91315948f3c9b370c9ae8453735e3ea2ff8a Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sat, 11 Feb 2017 23:16:05 -0800 Subject: [PATCH 08/42] Now generating test wrappers New wrappers invoke test methods within target widget on a test-by-test basis. test_atc is accomodating, test_frame not yet addressed. cored out most of TestWidget due to no-longer-necessary. --- unittests/atc.py | 102 +++++++++++++++--------------------------- unittests/test_atc.py | 12 +++-- 2 files changed, 45 insertions(+), 69 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 483f2ad12e..ebd3309849 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -63,6 +63,11 @@ def __init__(self): # the mainloop was running, but this caused issues on some linux # distributions. # Ultimately I'd like to minimize as much waiting as feasibly possible + + # this only gets invoked if the test arget is a frame object, in which case the app can post events directly + # to the test target + def GetTestTarget(self): + return self def OnCommence(self, evt): assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding @@ -76,81 +81,32 @@ def OnCommence(self, evt): # start test sequence evt = TestEvent() + evt.case = wx.GetApp().case wx.PostEvent(self, evt) - def OnWatchDog(self, evt): - # this handler is not used yet. - assert 0, "Watchdog failure." def OnTest(self, evt): - # find test case and try it - try: - testname = self.schedule[TestEvent.caseiter] - TestEvent.caseiter += 1 - except IndexError: - # well either some one made schedule some funky object - # or the end of the schedule has been reached. - # for now pretend its a good thing: - print("Reached end of schedule") - self.WrapUp() - return - - try: - testfunc = getattr(self, "test_%s" % testname) - except AttributeError: - print("Missing test method for: %s" % testname, file = sys.stderr) - sys.exit(1) - - try: - print("Testing: %s" % testname) - testfunc() - except Exception as e: - import traceback - _, _, tb = sys.exc_info() - traceback.print_tb(tb) - tb_info = traceback.extract_tb(tb) - - filename, line, func, text = tb_info[-1] - print("Additional Info:\nFile: %s\nLine: %s\nFunc: %s\nText: %s" % (filename, line, func, text), file = sys.stderr) - sys.exit(1) + testfunc = getattr(self, evt.case) + + print("Testing: %s" % evt.case) + testfunc() def TestDone(self, passed = True): # record test status and invoke next test if passed: - self.__results.append(".") - else: - self.__results.append("F") - - evt = TestEvent() - wx.PostEvent(self, evt) - - - def Assert(self, expr, message = "An Assertion occurred"): - # well, misnomer. Is a psuedo assert - # use this method when ensuring the test can continue - # in most test-failure scenarios you should use TestDone(False) - if not expr: - print(message, file = sys.stderr) - raise Exception("Test Failure") - - def WrapUp(self): - print("Results: %s" % "".join(self.__results)) - - if "F" in self.__results: - print("Test(s) have failed", file = sys.stderr) - sys.exit(1) - - else: - # close peacefully for window in wx.GetTopLevelWindows(): window.Close() + else: + wx.GetApp().GetMainLoop().Exit(30) + def CreateApp(frame): class TestApp(wx.App): def OnInit(self): - f = frame() - f.Show(True) + self.frame = frame() + self.frame.Show(True) return True + return TestApp def CreateFrame(widget): @@ -166,8 +122,20 @@ def __init__(self): self.SetSizer(sizer) sizer.Layout() + # this will only be invoked if the test widget is not a frame. + def GetTestTarget(self): + return self.widget + return BaseTestFrame +def CreateTestMethod(app, case): + def test_func(obj): + a = app() + a.case = case + a.MainLoop() + + return test_func + def CreateATC(widget): # if widget is not instance of TestFrame, generate a quick frame # to house the widget @@ -182,15 +150,17 @@ def CreateATC(widget): tlw = widget app = CreateApp(tlw) - app.targetwidget = widget class ApplicationTestCase(unittest.TestCase): - # generate test case wrappers? - def test_application(self): - a = app() - a.MainLoop() + pass + + methods = [meth for meth in dir(widget) if (meth.startswith("test_") and callable(getattr(widget, meth)))] + print(methods) + for meth in methods: + test_func = CreateTestMethod(app, meth) + setattr(ApplicationTestCase, meth, test_func) return ApplicationTestCase if __name__ == "__main__": - main() + pass diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 9e1ddafce0..29f3ec08c6 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -12,10 +12,16 @@ def __init__(self, parent): self.SetBackgroundColour(wx.BLUE) - def test_dummy(self): - self.TestDone(True) + def test_pass(self): + print("Passing test") + self.TestDone() + + def test_fail(self): + print("Failing test") + self.TestDone(False) testcase = atc.CreateATC(PanelColorChangeTester) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() + \ No newline at end of file From 6abef14be29e99699ffb8710c05844d9bf2ff993 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sat, 11 Feb 2017 23:48:06 -0800 Subject: [PATCH 09/42] Adapted test_frame to new test case style works as anticipated. --- unittests/test_frame.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index f1b7446a99..bc15d79af0 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -72,28 +72,33 @@ def __init__(self): # enforce a strict schedule self.schedule = ("iconize", "restore", "maximize", "restore") - def test_iconize(self): + def test_iconize_restore(self): self.Iconize() wx.CallLater(250, self.Ensure, "Iconized") - def test_maximize(self): + def test_maximize_restore(self): self.Maximize() wx.CallLater(250, self.Ensure, "Maximized") - def test_restore(self): - self.Restore() - wx.CallLater(250, self.Ensure, "Restored") - def Ensure(self, ensurable): if ensurable == "Iconized": - self.TestDone(self.IsIconized()) + if not self.IsIconized(): + self.TestDone(False) + self.Restore() + wx.CallLater(250, self.Ensure, "Restored") + elif ensurable == "Maximized": - self.TestDone(self.IsMaximized()) + if not self.IsMaximized(): + self.TestDone(False) + self.Restore() + wx.CallLater(250, self.Ensure, "Restored") + + elif ensurable == "Restored": self.TestDone(not self.IsIconized() and not self.IsMaximized()) -tc = atc.CreateATC(widget = FrameRestoreTester) +tc = atc.CreateATC(FrameRestoreTester) #--------------------------------------------------------------------------- From 590c399156fb244888ccd69b1769440beef7679d Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sat, 11 Feb 2017 23:54:16 -0800 Subject: [PATCH 10/42] Demorgan's law and ensuring a non-zero exit on failure. --- unittests/atc.py | 4 +++- unittests/test_frame.py | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index ebd3309849..f4e140fb23 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -97,7 +97,9 @@ def TestDone(self, passed = True): for window in wx.GetTopLevelWindows(): window.Close() else: - wx.GetApp().GetMainLoop().Exit(30) + # Clean exit pending investigation + # wx.GetApp().GetMainLoop().Exit(30) + sys.exit(1) def CreateApp(frame): diff --git a/unittests/test_frame.py b/unittests/test_frame.py index bc15d79af0..629a950060 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -69,9 +69,6 @@ def __init__(self): atc.TestWidget.__init__(self) self.SetLabel("Frame Restore Test") - # enforce a strict schedule - self.schedule = ("iconize", "restore", "maximize", "restore") - def test_iconize_restore(self): self.Iconize() wx.CallLater(250, self.Ensure, "Iconized") @@ -95,7 +92,7 @@ def Ensure(self, ensurable): elif ensurable == "Restored": - self.TestDone(not self.IsIconized() and not self.IsMaximized()) + self.TestDone(not (self.IsIconized() or self.IsMaximized())) tc = atc.CreateATC(FrameRestoreTester) From 40cd8cbf83c3e02d1fc5b6ba1c0061c9eedda2cc Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sun, 12 Feb 2017 11:45:17 -0800 Subject: [PATCH 11/42] Add decorator method for catching exceptions and aborting --- unittests/atc.py | 15 +++++++++++++-- unittests/test_atc.py | 10 +++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index f4e140fb23..c019eaff94 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -24,6 +24,7 @@ __version__ = "0.0.1" +import functools import os import sys import unittest @@ -101,6 +102,16 @@ def TestDone(self, passed = True): # wx.GetApp().GetMainLoop().Exit(30) sys.exit(1) +def TestDependent(func): + @functools.wraps(func) + def method(*args, **kwargs): + try: + func(*args, **kwargs) + except Exception as e: + print("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) + # give full stack trace + args[0].TestDone(False) + return method def CreateApp(frame): class TestApp(wx.App): @@ -164,5 +175,5 @@ class ApplicationTestCase(unittest.TestCase): return ApplicationTestCase -if __name__ == "__main__": - pass + + diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 29f3ec08c6..e8cdaf2c6d 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -1,9 +1,6 @@ from unittests import atc import wx import unittest -import random - -random.seed() class PanelColorChangeTester(wx.Panel, atc.TestWidget): def __init__(self, parent): @@ -20,8 +17,15 @@ def test_fail(self): print("Failing test") self.TestDone(False) + @atc.TestDependent + def test_abort(self): + print("Aborting test case") + raise AttributeError("Unbound attribute error") + + testcase = atc.CreateATC(PanelColorChangeTester) if __name__ == "__main__": + testcase.test_abort(testcase) unittest.main() \ No newline at end of file From 612950084eb76079c998b59542dca812471899d7 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sun, 12 Feb 2017 22:03:03 -0800 Subject: [PATCH 12/42] Changed decorator method name to better reflect its role --- unittests/atc.py | 4 +--- unittests/test_atc.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index c019eaff94..2ba1ffc822 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -78,8 +78,6 @@ def OnCommence(self, evt): if not hasattr(self, "schedule"): self.schedule = [member[5:] for member in dir(self) if member.startswith("test_")] - self.__results = [] - # start test sequence evt = TestEvent() evt.case = wx.GetApp().case @@ -102,7 +100,7 @@ def TestDone(self, passed = True): # wx.GetApp().GetMainLoop().Exit(30) sys.exit(1) -def TestDependent(func): +def TestCritical(func): @functools.wraps(func) def method(*args, **kwargs): try: diff --git a/unittests/test_atc.py b/unittests/test_atc.py index e8cdaf2c6d..123934c64c 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -17,15 +17,14 @@ def test_fail(self): print("Failing test") self.TestDone(False) - @atc.TestDependent + @atc.TestCritical def test_abort(self): print("Aborting test case") - raise AttributeError("Unbound attribute error") + assert 0 testcase = atc.CreateATC(PanelColorChangeTester) if __name__ == "__main__": - testcase.test_abort(testcase) unittest.main() \ No newline at end of file From 7bea6c8df1e312fb833ca17803c7a96fea07df00 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 13 Feb 2017 18:40:33 -0800 Subject: [PATCH 13/42] Ensure method decorations are preserved through to the wrappedm ethod --- unittests/atc.py | 24 ++++++------------------ unittests/test_atc.py | 8 ++++++-- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 2ba1ffc822..7ad9f70f88 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -35,19 +35,6 @@ TestEvent.caseiter = 0 -""" -class TestWidget: - # provides some useful methods for use within test widgets. - # it is recommended to derive test objects from this class in addtion to the - # relevate wx class - - # provide methods for: ensuring application closes (with and without an error code) - # standardized assertion - # etc. - - # should this class provide methods for event handling (forcing derivation from wx.EventHandler)? -""" - class TestWidget: def __init__(self): assert isinstance(self, wx.EvtHandler), "Test widget needs to be an event handler" @@ -73,11 +60,6 @@ def GetTestTarget(self): def OnCommence(self, evt): assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding # commencing with the timer. - - # ensure the test schedule is set up, create it if it does not exist - if not hasattr(self, "schedule"): - self.schedule = [member[5:] for member in dir(self) if member.startswith("test_")] - # start test sequence evt = TestEvent() evt.case = wx.GetApp().case @@ -169,6 +151,12 @@ class ApplicationTestCase(unittest.TestCase): print(methods) for meth in methods: test_func = CreateTestMethod(app, meth) + # ensure any other deocrated data is preserved: + basemeth = getattr(widget, meth) + for attr in dir(basemeth): + if not hasattr(test_func, attr): + setattr(test_func, attr, getattr(basemeth, attr)) + setattr(ApplicationTestCase, meth, test_func) return ApplicationTestCase diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 123934c64c..9d861cd6d5 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -2,7 +2,7 @@ import wx import unittest -class PanelColorChangeTester(wx.Panel, atc.TestWidget): +class ATCPanel(wx.Panel, atc.TestWidget): def __init__(self, parent): wx.Panel.__init__(self, parent, wx.NewId()) atc.TestWidget.__init__(self) @@ -21,9 +21,13 @@ def test_fail(self): def test_abort(self): print("Aborting test case") assert 0 + + @unittest.skip("reasons") + def test_skip(self): + return -testcase = atc.CreateATC(PanelColorChangeTester) +testcase = atc.CreateATC(ATCPanel) if __name__ == "__main__": unittest.main() From 294f110139c7b40370d85d2ffc46f7e092c2bb2f Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Tue, 14 Feb 2017 17:10:02 +0000 Subject: [PATCH 14/42] communicate critical exceptions to outside of app --- unittests/atc.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 2ba1ffc822..8adbe36c5e 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -64,7 +64,7 @@ def __init__(self): # the mainloop was running, but this caused issues on some linux # distributions. # Ultimately I'd like to minimize as much waiting as feasibly possible - + # this only gets invoked if the test arget is a frame object, in which case the app can post events directly # to the test target def GetTestTarget(self): @@ -108,7 +108,10 @@ def method(*args, **kwargs): except Exception as e: print("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) # give full stack trace - args[0].TestDone(False) + wx.GetApp().exception = e + for window in wx.GetTopLevelWindows(): + window.Close() + return method def CreateApp(frame): @@ -145,6 +148,12 @@ def test_func(obj): a.case = case a.MainLoop() + if hasattr(a, "exception"): + raise a.exception + + elif hasattr(a, "errorcode"): + sys.exit(a.errorcode) + return test_func def CreateATC(widget): @@ -172,6 +181,3 @@ class ApplicationTestCase(unittest.TestCase): setattr(ApplicationTestCase, meth, test_func) return ApplicationTestCase - - - From 570f82dfc8f13380edfd2009849fb16181ab40e9 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Tue, 14 Feb 2017 17:28:28 +0000 Subject: [PATCH 15/42] fixed TestError exception --- unittests/atc.py | 11 +++++------ unittests/test_atc.py | 8 ++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index bd899c3e80..029c3cbde4 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -33,7 +33,8 @@ TestEvent, EVT_TEST = wx.lib.newevent.NewEvent() -TestEvent.caseiter = 0 +class TestError(Exception): + pass class TestWidget: def __init__(self): @@ -78,9 +79,7 @@ def TestDone(self, passed = True): for window in wx.GetTopLevelWindows(): window.Close() else: - # Clean exit pending investigation - # wx.GetApp().GetMainLoop().Exit(30) - sys.exit(1) + raise TestError("A test failed.") def TestCritical(func): @functools.wraps(func) @@ -93,7 +92,7 @@ def method(*args, **kwargs): wx.GetApp().exception = e for window in wx.GetTopLevelWindows(): window.Close() - + return method def CreateApp(frame): @@ -165,7 +164,7 @@ class ApplicationTestCase(unittest.TestCase): for attr in dir(basemeth): if not hasattr(test_func, attr): setattr(test_func, attr, getattr(basemeth, attr)) - + setattr(ApplicationTestCase, meth, test_func) return ApplicationTestCase diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 9d861cd6d5..36be23b9f5 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -7,21 +7,22 @@ def __init__(self, parent): wx.Panel.__init__(self, parent, wx.NewId()) atc.TestWidget.__init__(self) - self.SetBackgroundColour(wx.BLUE) - def test_pass(self): print("Passing test") self.TestDone() + @unittest.expectedFailure + @atc.TestCritical def test_fail(self): print("Failing test") self.TestDone(False) + @unittest.expectedFailure @atc.TestCritical def test_abort(self): print("Aborting test case") assert 0 - + @unittest.skip("reasons") def test_skip(self): return @@ -31,4 +32,3 @@ def test_skip(self): if __name__ == "__main__": unittest.main() - \ No newline at end of file From 022da563e467348d08fba43c23f4b6144d297964 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Tue, 14 Feb 2017 17:47:19 +0000 Subject: [PATCH 16/42] updated documentation at top of module --- unittests/atc.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 029c3cbde4..efe8bcb95d 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -1,28 +1,34 @@ # Samuel Dunn # Application Test Case -# allows testing wx features within the context of an application +# Allows testing wx features within the context of an application + +# Benefits: As stated, allows unit testing within a full App mainloop. +# Allows developers to quickly construct a test case by developing a +# single widget +# TestCase class is generated from the widget provided to atc. The +# generated TestCase class mimics the widget to allow pytest +# to behave normally. This allows unsquashed testing (jamming +# plural tests into one runtime) +# Generated TestCase class should play nice with all unittest features +# Generated TestCase classes are unique, allowing plural per test +# module + +# Drawbacks: Generated test case class has to be applied to a __main__ module +# global scope member. Need to look into unittest.TestDiscorvery +# to correct this behavior +# Code is currently a bit sloppy, Revisions will be made before a +# formal PR to improve documentation and clean up the code. +# -# Drawbacks: Current revision doesn't play into unittest's framework very well -# mimicking it more than anything -# Current revision only supports handling test events through a top level Frame -# derived from TestFrame, alternatives are being considered -# Only one TestCase class per module +# TODO: +# Ensure full TestCase API is available within the app +# automatically apply TestCritical decorator to test_ methods in widget +# Ensure TestCritical decorator plays nice with preserving other decorations +# such that decoration order does not matter +# Add stack trace printouts upon TestDone(False) or TestCritical exception -# Benefits: Allows unittesting features that suffer from only synthesizing an active event loop. -# TODO: -# Create a decorator method to identify methods that are fully self sufficient tests, so TestDone -# can be called implicitly -# On The other hand, its only truly useful to know if the test completes as part of the method -# A decorator for describing methods that are a part of a test sequence, but not the entire sequence -# could be useful -# Explore TestWidget possibilty -# Explore commencing test schedule various ways (depending on OS) -# Change ApplicationTestCase to be a TestCase class generator, allowing plural testcase classes per module -# explore potential for a meta-class that decorates test related methods to ensure test failure on unhandled -# exception (related to first couple TODOs) - -__version__ = "0.0.1" +__version__ = "0.0.2" import functools import os From f1b8aea1d7f314dc95e07179a9e16d04ad5fb1b7 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Tue, 14 Feb 2017 17:54:00 +0000 Subject: [PATCH 17/42] Added TestCritical decorator to ensure, to cover TestError change to TestDone --- unittests/test_frame.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 629a950060..f24b453f0d 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -77,19 +77,20 @@ def test_maximize_restore(self): self.Maximize() wx.CallLater(250, self.Ensure, "Maximized") + @atc.TestCritical def Ensure(self, ensurable): if ensurable == "Iconized": if not self.IsIconized(): self.TestDone(False) self.Restore() wx.CallLater(250, self.Ensure, "Restored") - + elif ensurable == "Maximized": if not self.IsMaximized(): self.TestDone(False) self.Restore() wx.CallLater(250, self.Ensure, "Restored") - + elif ensurable == "Restored": self.TestDone(not (self.IsIconized() or self.IsMaximized())) From 9fbb399db03afa342bcced6a43783763e2614747 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Wed, 15 Feb 2017 12:10:33 -0800 Subject: [PATCH 18/42] added 5 minute watchdog to TestWidget. A runtime error is raised should the timeout occur --- unittests/atc.py | 8 ++++++++ unittests/test_atc.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/unittests/atc.py b/unittests/atc.py index efe8bcb95d..048335c63e 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -72,6 +72,14 @@ def OnCommence(self, evt): evt.case = wx.GetApp().case wx.PostEvent(self, evt) + # set a watchdog incase of test error + wx.CallLater(300000, self.OnWatchdog) # 5 minutes in millis + + def OnWatchdog(self): + print("Test Timeout!!!") + wx.GetApp().exception = RuntimeError("Watchdog timed out") + for window in wx.GetTopLevelWindows(): + window.Close() def OnTest(self, evt): testfunc = getattr(self, evt.case) diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 36be23b9f5..8415bc0fa1 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -23,6 +23,11 @@ def test_abort(self): print("Aborting test case") assert 0 + @unittest.expectedFailure + @atc.TestCritical + def test_timeout(self): + print("Letting app lo0se") + @unittest.skip("reasons") def test_skip(self): return From 01af107df5e5efed1676336a87796782c04cf4c9 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Wed, 15 Feb 2017 12:50:00 -0800 Subject: [PATCH 19/42] Swapped from directly using a timer to CallLater, removed unnecessarily long timeout test. --- unittests/atc.py | 22 ++++++++++------------ unittests/test_atc.py | 5 ----- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 048335c63e..f72f16e503 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -46,25 +46,23 @@ class TestWidget: def __init__(self): assert isinstance(self, wx.EvtHandler), "Test widget needs to be an event handler" - self.__timer = wx.Timer(self, wx.NewId()) - - self.Bind(wx.EVT_TIMER, self.OnCommence, self.__timer) self.Bind(EVT_TEST, self.OnTest) - self.__timer.StartOnce(500) # Wait for the application mainloop to start and commence testing. - # while 500 milliseconds should be more than plenty, this is still - # clumsy and I don't like it. - # Initially I had an initial TestEvent posted to be processed when - # the mainloop was running, but this caused issues on some linux - # distributions. - # Ultimately I'd like to minimize as much waiting as feasibly possible + wx.CallLater(500, self.OnCommence) + # Wait for the application mainloop to start and commence testing. + # while 500 milliseconds should be more than plenty, this is still + # clumsy and I don't like it. + # Initially I had an initial TestEvent posted to be processed when + # the mainloop was running, but this caused issues on some linux + # distributions. + # Ultimately I'd like to minimize as much waiting as feasibly possible # this only gets invoked if the test arget is a frame object, in which case the app can post events directly # to the test target def GetTestTarget(self): return self - def OnCommence(self, evt): + def OnCommence(self): assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding # commencing with the timer. # start test sequence @@ -170,7 +168,7 @@ class ApplicationTestCase(unittest.TestCase): pass methods = [meth for meth in dir(widget) if (meth.startswith("test_") and callable(getattr(widget, meth)))] - print(methods) + for meth in methods: test_func = CreateTestMethod(app, meth) # ensure any other deocrated data is preserved: diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 8415bc0fa1..36be23b9f5 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -23,11 +23,6 @@ def test_abort(self): print("Aborting test case") assert 0 - @unittest.expectedFailure - @atc.TestCritical - def test_timeout(self): - print("Letting app lo0se") - @unittest.skip("reasons") def test_skip(self): return From cba6425eeb9eecce129b34c63d1d175b82ba5c6d Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 10:22:10 -0800 Subject: [PATCH 20/42] swapped from TestDone to testFailed and testPassed --- unittests/atc.py | 16 +++++++++------- unittests/test_atc.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index f72f16e503..2732c9b206 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -85,14 +85,16 @@ def OnTest(self, evt): print("Testing: %s" % evt.case) testfunc() - def TestDone(self, passed = True): - # record test status and invoke next test - if passed: - for window in wx.GetTopLevelWindows(): - window.Close() - else: - raise TestError("A test failed.") + def testPassed(self): + for window in wx.GetTopLevelWindows(): + window.Close() + def testFailed(self, errmsg = "A test failed."): + # do not rely on testCritical being applied to an above method + wx.GetApp().exception = TestError(errmsg) + for window in wx.GetTopLevelWindows(): + window.Close() + def TestCritical(func): @functools.wraps(func) def method(*args, **kwargs): diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 36be23b9f5..b1031bec11 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -9,13 +9,13 @@ def __init__(self, parent): def test_pass(self): print("Passing test") - self.TestDone() + self.testPassed() @unittest.expectedFailure @atc.TestCritical def test_fail(self): print("Failing test") - self.TestDone(False) + self.testFailed() @unittest.expectedFailure @atc.TestCritical From f76042e89e823e71611445c51876d33863fcc925 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 10:37:23 -0800 Subject: [PATCH 21/42] Changed TestCritical to testCritical, added informative docstring, moved it to top of module for in-module usage. the exception-protection is now applied to OnTest, meaning that the test widget's test_ methods will automatically have testCritical's protection --- unittests/atc.py | 41 ++++++++++++++++++++++------------------- unittests/test_atc.py | 2 -- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 2732c9b206..04d1049caa 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -42,6 +42,27 @@ class TestError(Exception): pass +def testCritical(func): + """ + Wraps the provided function to ensure that uncaught exceptions + will end execution. + This is done by closing all top level windows and allowing the exception + to be re-raised once the main event loop ends. + """ + @functools.wraps(func) + def method(*args, **kwargs): + try: + func(*args, **kwargs) + except Exception as e: + print("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) + # give full stack trace + wx.GetApp().exception = e + for window in wx.GetTopLevelWindows(): + window.Close() + + return method + + class TestWidget: def __init__(self): assert isinstance(self, wx.EvtHandler), "Test widget needs to be an event handler" @@ -57,11 +78,6 @@ def __init__(self): # distributions. # Ultimately I'd like to minimize as much waiting as feasibly possible - # this only gets invoked if the test arget is a frame object, in which case the app can post events directly - # to the test target - def GetTestTarget(self): - return self - def OnCommence(self): assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding # commencing with the timer. @@ -79,6 +95,7 @@ def OnWatchdog(self): for window in wx.GetTopLevelWindows(): window.Close() + @testCritical # automatically apply exception blocking to test_ methods.. indirectly def OnTest(self, evt): testfunc = getattr(self, evt.case) @@ -95,20 +112,6 @@ def testFailed(self, errmsg = "A test failed."): for window in wx.GetTopLevelWindows(): window.Close() -def TestCritical(func): - @functools.wraps(func) - def method(*args, **kwargs): - try: - func(*args, **kwargs) - except Exception as e: - print("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) - # give full stack trace - wx.GetApp().exception = e - for window in wx.GetTopLevelWindows(): - window.Close() - - return method - def CreateApp(frame): class TestApp(wx.App): def OnInit(self): diff --git a/unittests/test_atc.py b/unittests/test_atc.py index b1031bec11..dc93e4de1d 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -12,13 +12,11 @@ def test_pass(self): self.testPassed() @unittest.expectedFailure - @atc.TestCritical def test_fail(self): print("Failing test") self.testFailed() @unittest.expectedFailure - @atc.TestCritical def test_abort(self): print("Aborting test case") assert 0 From 4f57a1cb32108af61f0dff8704943b3bb6eaa081 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 11:13:25 -0800 Subject: [PATCH 22/42] Changed test launch as per @RobinD42's suggestions. --- unittests/atc.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 04d1049caa..84cb312cc5 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -69,14 +69,11 @@ def __init__(self): self.Bind(EVT_TEST, self.OnTest) - wx.CallLater(500, self.OnCommence) - # Wait for the application mainloop to start and commence testing. - # while 500 milliseconds should be more than plenty, this is still - # clumsy and I don't like it. - # Initially I had an initial TestEvent posted to be processed when - # the mainloop was running, but this caused issues on some linux - # distributions. - # Ultimately I'd like to minimize as much waiting as feasibly possible + if "__WXGTK__" in wx.PlatformInfo: + self.Bind(wx.EVT_WINDOW_CREATE, self.OnLnxStart) + + else: + wx.CallAfter(self.OnCommence) def OnCommence(self): assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding @@ -95,6 +92,13 @@ def OnWatchdog(self): for window in wx.GetTopLevelWindows(): window.Close() + def OnLnxStart(self, evt): + """ + Invoked on linux systems to signal mainloop readiness + """ + wx.CallAfter(self.OnCommence) + evt.Skip() + @testCritical # automatically apply exception blocking to test_ methods.. indirectly def OnTest(self, evt): testfunc = getattr(self, evt.case) From 0a0a356c4cb30f29f60e501a0653b90688ae5810 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 11:14:20 -0800 Subject: [PATCH 23/42] Update test_frame to work with recent atc changes --- unittests/test_frame.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index f24b453f0d..06b1c4cde2 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -77,23 +77,26 @@ def test_maximize_restore(self): self.Maximize() wx.CallLater(250, self.Ensure, "Maximized") - @atc.TestCritical + @atc.testCritical def Ensure(self, ensurable): if ensurable == "Iconized": if not self.IsIconized(): - self.TestDone(False) + self.testFailed("Frame failed to iconize") self.Restore() wx.CallLater(250, self.Ensure, "Restored") elif ensurable == "Maximized": if not self.IsMaximized(): - self.TestDone(False) + self.testFailed("Frame failed to maximize") self.Restore() wx.CallLater(250, self.Ensure, "Restored") elif ensurable == "Restored": - self.TestDone(not (self.IsIconized() or self.IsMaximized())) + if (not (self.IsIconized or self.IsMaximized())): + self.testPassed() + else: + self.testFailed("Window is not restored.") tc = atc.CreateATC(FrameRestoreTester) From a3bf5aab4da737f142c84ffd2c215eb00b35a423 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 11:25:54 -0800 Subject: [PATCH 24/42] Changed CreateATC to createATC --- unittests/atc.py | 2 +- unittests/test_atc.py | 2 +- unittests/test_frame.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 84cb312cc5..5e50bcd9d0 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -158,7 +158,7 @@ def test_func(obj): return test_func -def CreateATC(widget): +def createATC(widget): # if widget is not instance of TestFrame, generate a quick frame # to house the widget assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" diff --git a/unittests/test_atc.py b/unittests/test_atc.py index dc93e4de1d..09e9e8562d 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -26,7 +26,7 @@ def test_skip(self): return -testcase = atc.CreateATC(ATCPanel) +testcase = atc.createATC(ATCPanel) if __name__ == "__main__": unittest.main() diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 06b1c4cde2..83b977483f 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -99,7 +99,7 @@ def Ensure(self, ensurable): self.testFailed("Window is not restored.") -tc = atc.CreateATC(FrameRestoreTester) +tc = atc.createATC(FrameRestoreTester) #--------------------------------------------------------------------------- From c4ce20761d768909ea4c653324f3008ccad0aafc Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 13:32:10 -0800 Subject: [PATCH 25/42] Added a ton of documentation. Rearranged some code to put user-exposed methods and classes first added dunders to methods not intended to be user-accessible. --- unittests/atc.py | 238 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 174 insertions(+), 64 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 5e50bcd9d0..599409df4c 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -62,91 +62,233 @@ def method(*args, **kwargs): return method +def createATC(widget): + """ + Creates and returns a class that derives unittest.TestCase the TestCase is generated from widget + IMPORTANT NOTE: In order for the returned class to be picked up by the default unittest TestDiscovery process + the returned class must be assigned to the main module's base level namespace + If widget is not top level (does not drive from wx.Frame) a container top level widget will be created + Additionally, if the widget is not top level generated frame object will attempt to pass + itself as a parameter to widget.__init__ (for assigning parent) + Be sure to expect this in such scenarios. + + The returned class will have test_ methods to match those of the widget class, all decorations are preserved + so it is perfectly valid to apply unittest decorators (such as unittest.expectedFailure) to the widget class + as such decorations will be reflected by the TestCase class methods and utilized during testing. + + Args: + widget: A widget *class* that derives from TestWidget. + + Returns: + unittest.TestCase derivation + + Example: + class Foo(wx.Frame, atc.TestWidget): + def __init__(self): + wx.Frame.__init__(self, None, wx.NewId()) + atc.TestWidget.__init__(self) + + def test_pass(self): + self.testPassed() + + @unittest.expectedFailure + def test_fail(self): + self.testFailed("Deliberate failure") + + FooTestCase = atc.createATC(Foo) + + # FooTestCase will have a methods test_pass and test_fail + # when run by pytest test_pass will pass and test_fail will + # xfail. + """ + assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" + + tlw = None + if not issubclass(widget, wx.Frame): + # need to stick this widget in a frame + tlw = __CreateFrame(widget) + else: + tlw = widget + + app = __CreateApp(tlw) + + class ApplicationTestCase(unittest.TestCase): + pass + + methods = [meth for meth in dir(widget) if (meth.startswith("test_") and callable(getattr(widget, meth)))] + + for meth in methods: + test_func = __CreateTestMethod(app, meth) + # ensure any other deocrated data is preserved: + basemeth = getattr(widget, meth) + for attr in dir(basemeth): + if not hasattr(test_func, attr): + setattr(test_func, attr, getattr(basemeth, attr)) + + setattr(ApplicationTestCase, meth, test_func) + + return ApplicationTestCase class TestWidget: + """ + Base Test Widget class. Widgets that are intended to be tested via ATC *MUST* derive this class + It is likewise expected that these widgets derive from some actual wx widget, at the minimum from wx.EvtHandler, the constructor for which must be called first + This class provides test sequencing to its derived class, most notably in starting the test automatically, and providing methods for termination. + For example usage please review unittests/test_atc. + """ def __init__(self): + """ + Ensures that class being instantiated also derives from wx.EvtHandler and prepares for auto-launch + Args: + self + """ assert isinstance(self, wx.EvtHandler), "Test widget needs to be an event handler" - self.Bind(EVT_TEST, self.OnTest) + self.Bind(EVT_TEST, self.__OnTest) if "__WXGTK__" in wx.PlatformInfo: - self.Bind(wx.EVT_WINDOW_CREATE, self.OnLnxStart) + self.Bind(wx.EVT_WINDOW_CREATE, self.__OnLnxStart) else: - wx.CallAfter(self.OnCommence) + wx.CallAfter(self.__OnCommence) + + def testPassed(self): + """ + Indicates that the currently running test has passed successfully. + The application will close clearly after this call, allowing unittesting to proceed + IMPORTANT NOTE: test sequences must terminate with either a call to this method or testFailed() + otherwise runtime may not close until the watchdog triggers (treating the test as a failure) + + Args: + None + + Returns: + None + """ + for window in wx.GetTopLevelWindows(): + window.Close() + + def testFailed(self, errmsg = "A test failed."): + """ + Indicates that the currently running test has failed. + The application will close after this call and an TestError exception will be raised + IMPORTANT NOTE: test sequences must terminate with either a call to this method or testPassed() + otherwise runtime may not close until the watchdog triggers (treating the test as a failure) + + Args: + errmsg: Message assigned to TestError exception. + Returns: + None + """ + + # do not rely on testCritical being applied to an above method + wx.GetApp().exception = TestError(errmsg) + for window in wx.GetTopLevelWindows(): + window.Close() + + def __OnCommence(self): + """ + Invoked to start test procedures. Invokation is handled by atc. + """ + # assert should exit properly if the mainloop is not running + assert wx.GetApp().IsMainLoopRunning(), "__OnCommence invoked before MainLoop was ready" - def OnCommence(self): - assert wx.GetApp().IsMainLoopRunning(), "Timer ended before mainloop started" # see above comment regarding - # commencing with the timer. # start test sequence evt = TestEvent() evt.case = wx.GetApp().case wx.PostEvent(self, evt) # set a watchdog incase of test error - wx.CallLater(300000, self.OnWatchdog) # 5 minutes in millis + wx.CallLater(300000, self.__OnWatchdog) # 5 minutes in millis - def OnWatchdog(self): + def __OnWatchdog(self): + """ + Invoked with a test hass taken more than 5 minutes to complete + """ print("Test Timeout!!!") wx.GetApp().exception = RuntimeError("Watchdog timed out") for window in wx.GetTopLevelWindows(): window.Close() - def OnLnxStart(self, evt): + def __OnLnxStart(self, evt): """ Invoked on linux systems to signal mainloop readiness """ - wx.CallAfter(self.OnCommence) + wx.CallAfter(self.__OnCommence) evt.Skip() @testCritical # automatically apply exception blocking to test_ methods.. indirectly - def OnTest(self, evt): + def __OnTest(self, evt): testfunc = getattr(self, evt.case) print("Testing: %s" % evt.case) testfunc() - - def testPassed(self): - for window in wx.GetTopLevelWindows(): - window.Close() - - def testFailed(self, errmsg = "A test failed."): - # do not rely on testCritical being applied to an above method - wx.GetApp().exception = TestError(errmsg) - for window in wx.GetTopLevelWindows(): - window.Close() -def CreateApp(frame): +def __CreateApp(frame_cls): + """ + Generates an app class that will create an instance of frame_cls on launch. + This method is utilized inside atc and probably should not be used otherwise + + Args: + frame_cls: Class that derives from wx.Frame + Returns: + wx.App derived class + """ class TestApp(wx.App): + """ a generated App class """ def OnInit(self): - self.frame = frame() + self.frame = frame_cls() self.frame.Show(True) return True return TestApp -def CreateFrame(widget): +def __CreateFrame(widget_cls): + """ + Creates a wx.Frame derivation to create an instance of widget_cls and sizes said instance to fill the frame + This method is utlized inside atc and probably should not be used otherwise + + Args: + widget_cls: Widget class (presumably the TestWidget) that will be initialized within the generated frame object + + Returns: + wx.Frame derived class to use for testing the widget. + """ # called when the test widget is not a frame. class BaseTestFrame(wx.Frame): + """ A generated Frame class """ def __init__(self): - wx.Frame.__init__(self, None, wx.NewId(), "%s Test Frame" % str(type(widget))) + wx.Frame.__init__(self, None, wx.NewId(), "Generated Test Frame") sizer = wx.BoxSizer() - self.widget = widget(self) # assumes need of parent. + self.widget = widget_cls(self) # assumes need of parent. sizer.Add(self.widget, 1, wx.EXPAND) self.SetSizer(sizer) sizer.Layout() - # this will only be invoked if the test widget is not a frame. - def GetTestTarget(self): - return self.widget - return BaseTestFrame -def CreateTestMethod(app, case): +def __CreateTestMethod(app_cls, case): + """ + Generates the test methods to assign to the ApplicationTestCase class. + Generated methods initialize an instance of app_cls, assign the intended test case and + launch the app. + Once the app has closed error conditions are checked and the function returns + + This method is utilized inside atc and should not be used otherwise. + + returned method objects are not redecorated (as critical class data is not present) + + Args: + app_cls: the wx.App derived class to instantiate + case: str indicating which test method to retrieve and invoke. + + Returns: + Method objects to launch an application for a given test case. + """ def test_func(obj): - a = app() + a = app_cls() a.case = case a.MainLoop() @@ -157,35 +299,3 @@ def test_func(obj): sys.exit(a.errorcode) return test_func - -def createATC(widget): - # if widget is not instance of TestFrame, generate a quick frame - # to house the widget - assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" - - tlw = None - if not issubclass(widget, wx.Frame): - # need to stick this widget in a frame - tlw = CreateFrame(widget) - - else: - tlw = widget - - app = CreateApp(tlw) - - class ApplicationTestCase(unittest.TestCase): - pass - - methods = [meth for meth in dir(widget) if (meth.startswith("test_") and callable(getattr(widget, meth)))] - - for meth in methods: - test_func = CreateTestMethod(app, meth) - # ensure any other deocrated data is preserved: - basemeth = getattr(widget, meth) - for attr in dir(basemeth): - if not hasattr(test_func, attr): - setattr(test_func, attr, getattr(basemeth, attr)) - - setattr(ApplicationTestCase, meth, test_func) - - return ApplicationTestCase From 44436070b8e68136c18757af41fb48f1afce0fb7 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 13:34:28 -0800 Subject: [PATCH 26/42] Added additional test case to test_atc, renamed test class name in test_frame --- unittests/test_atc.py | 18 +++++++++++++++++- unittests/test_frame.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 09e9e8562d..4145ca521a 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -23,10 +23,26 @@ def test_abort(self): @unittest.skip("reasons") def test_skip(self): + self.testFailed("test_skip was invoked!") + + @unittest.skip("Takes too long for regular testing") + def test_watchdog(self): + print("Letting application loose") return +class ATCFrame(wx.Frame, atc.TestWidget): + def __init__(self): + wx.Frame.__init__(self, None, wx.NewId(), "ATC Test Frame") + atc.TestWidget.__init__(self) + + def test_pass(self): + self.testPassed() + + @unittest.expectedFailure + def test_fail(self): + self.testFailed("Deliberate test failure") -testcase = atc.createATC(ATCPanel) +atc_BasicTests = atc.createATC(ATCPanel) if __name__ == "__main__": unittest.main() diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 83b977483f..3f7146fd46 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -99,7 +99,7 @@ def Ensure(self, ensurable): self.testFailed("Window is not restored.") -tc = atc.createATC(FrameRestoreTester) +frame_RestoreTests = atc.createATC(FrameRestoreTester) #--------------------------------------------------------------------------- From 124d214ec987c15235dedf5336dc7d2922807bcf Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 14:34:22 -0800 Subject: [PATCH 27/42] Create ATC for ATCFrame (oops) --- unittests/test_atc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 4145ca521a..c6bd6dd0ce 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -43,6 +43,7 @@ def test_fail(self): self.testFailed("Deliberate test failure") atc_BasicTests = atc.createATC(ATCPanel) +atc_FrameTests = atc.createATC(ATCFrame) if __name__ == "__main__": unittest.main() From e80487df428f9b91d7f194316e86a7ae47a8c428 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 16:36:09 -0800 Subject: [PATCH 28/42] fixed conditional --- unittests/test_frame.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 3f7146fd46..dfb532b59f 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -83,17 +83,17 @@ def Ensure(self, ensurable): if not self.IsIconized(): self.testFailed("Frame failed to iconize") self.Restore() - wx.CallLater(250, self.Ensure, "Restored") + wx.CallLater(500, self.Ensure, "Restored") elif ensurable == "Maximized": if not self.IsMaximized(): self.testFailed("Frame failed to maximize") self.Restore() - wx.CallLater(250, self.Ensure, "Restored") + wx.CallLater(500, self.Ensure, "Restored") elif ensurable == "Restored": - if (not (self.IsIconized or self.IsMaximized())): + if (not (self.IsIconized() or self.IsMaximized())): self.testPassed() else: self.testFailed("Window is not restored.") From c7d2e521209eb253199907470eec7ae20dcf385b Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 17:41:12 -0800 Subject: [PATCH 29/42] corrected for timing issue that arose in linux --- unittests/test_frame.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unittests/test_frame.py b/unittests/test_frame.py index dfb532b59f..75ed4ef5e8 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -70,6 +70,12 @@ def __init__(self): self.SetLabel("Frame Restore Test") def test_iconize_restore(self): + # for some reason, gtk seems to need a bit more time after + # launch to successfully iconize. + if "__WXGTK__" in wx.PlatformInfo: + wx.CallLater(500, self.lnx_iconize_restore) + return + self.Iconize() wx.CallLater(250, self.Ensure, "Iconized") @@ -77,6 +83,11 @@ def test_maximize_restore(self): self.Maximize() wx.CallLater(250, self.Ensure, "Maximized") + @atc.testCritical + def lnx_iconize_restore(self): + self.Iconize() + wx.CallLater(250, self.Ensure, "Iconized") + @atc.testCritical def Ensure(self, ensurable): if ensurable == "Iconized": From c4aa4def47760af10cc85d05f3693f2cffb860bf Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 16 Feb 2017 17:52:46 -0800 Subject: [PATCH 30/42] Provide access to TestCase class instance through new TestWidget method getTestCase --- unittests/atc.py | 13 +++++++++++++ unittests/test_atc.py | 4 ++++ unittests/test_frame.py | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 599409df4c..8349309498 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -186,6 +186,18 @@ def testFailed(self, errmsg = "A test failed."): for window in wx.GetTopLevelWindows(): window.Close() + def getTestCase(self): + """ + Returns the encompassing TestCase class, which can then be used for its various + testing methods. + + Args: + None + Returns: + Encompassing ATC class. + """ + return wx.GetApp().testcase + def __OnCommence(self): """ Invoked to start test procedures. Invokation is handled by atc. @@ -290,6 +302,7 @@ def __CreateTestMethod(app_cls, case): def test_func(obj): a = app_cls() a.case = case + a.testcase = obj a.MainLoop() if hasattr(a, "exception"): diff --git a/unittests/test_atc.py b/unittests/test_atc.py index c6bd6dd0ce..995d2fc2cf 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -30,6 +30,10 @@ def test_watchdog(self): print("Letting application loose") return + def test_testcase(self): + self.getTestCase() + self.testPassed() + class ATCFrame(wx.Frame, atc.TestWidget): def __init__(self): wx.Frame.__init__(self, None, wx.NewId(), "ATC Test Frame") diff --git a/unittests/test_frame.py b/unittests/test_frame.py index 75ed4ef5e8..e43a311b21 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -94,13 +94,13 @@ def Ensure(self, ensurable): if not self.IsIconized(): self.testFailed("Frame failed to iconize") self.Restore() - wx.CallLater(500, self.Ensure, "Restored") + wx.CallLater(250, self.Ensure, "Restored") elif ensurable == "Maximized": if not self.IsMaximized(): self.testFailed("Frame failed to maximize") self.Restore() - wx.CallLater(500, self.Ensure, "Restored") + wx.CallLater(250, self.Ensure, "Restored") elif ensurable == "Restored": From 2c8f11d8da28af7dd3f152d33762597d755288a8 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sun, 19 Feb 2017 23:51:12 -0800 Subject: [PATCH 31/42] Conformed Argument name to denote argument is a class --- unittests/atc.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 8349309498..4b83ca946f 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -62,7 +62,7 @@ def method(*args, **kwargs): return method -def createATC(widget): +def createATC(widget_cls): """ Creates and returns a class that derives unittest.TestCase the TestCase is generated from widget IMPORTANT NOTE: In order for the returned class to be picked up by the default unittest TestDiscovery process @@ -101,26 +101,25 @@ def test_fail(self): # when run by pytest test_pass will pass and test_fail will # xfail. """ - assert issubclass(widget, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" - + assert issubclass(widget_cls, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" tlw = None - if not issubclass(widget, wx.Frame): + if not issubclass(widget_cls, wx.Frame): # need to stick this widget in a frame - tlw = __CreateFrame(widget) + tlw = __CreateFrame(widget_cls) else: - tlw = widget + tlw = widget_cls app = __CreateApp(tlw) class ApplicationTestCase(unittest.TestCase): pass - methods = [meth for meth in dir(widget) if (meth.startswith("test_") and callable(getattr(widget, meth)))] + methods = [meth for meth in dir(widget_cls) if (meth.startswith("test_") and callable(getattr(widget_cls, meth)))] for meth in methods: test_func = __CreateTestMethod(app, meth) # ensure any other deocrated data is preserved: - basemeth = getattr(widget, meth) + basemeth = getattr(widget_cls, meth) for attr in dir(basemeth): if not hasattr(test_func, attr): setattr(test_func, attr, getattr(basemeth, attr)) From 8c4b396ff5c9fc2379bb911305c2e6dd45c3fdab Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sun, 19 Feb 2017 23:53:02 -0800 Subject: [PATCH 32/42] temporarily removed support for testing dialog derivatives --- unittests/atc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unittests/atc.py b/unittests/atc.py index 4b83ca946f..9ec224e851 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -102,6 +102,7 @@ def test_fail(self): # xfail. """ assert issubclass(widget_cls, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" + assert not issubclass(widget_cls, wx.Dialog), "Support for wx.Dialog derivatives suspended." tlw = None if not issubclass(widget_cls, wx.Frame): # need to stick this widget in a frame From 1806a89916e8981d39e9ab0b0b468d2c5870140a Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 20 Feb 2017 12:40:04 -0800 Subject: [PATCH 33/42] Support wx.Dialog derivatives by launching them modeless. Normal test procedures take care of the rest --- unittests/atc.py | 17 +++++++++++------ unittests/test_atc.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 9ec224e851..1236da04bd 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -102,7 +102,7 @@ def test_fail(self): # xfail. """ assert issubclass(widget_cls, TestWidget), "Testing requires the tested widget to derive from TestWidget for now" - assert not issubclass(widget_cls, wx.Dialog), "Support for wx.Dialog derivatives suspended." + tlw = None if not issubclass(widget_cls, wx.Frame): # need to stick this widget in a frame @@ -272,12 +272,17 @@ class BaseTestFrame(wx.Frame): def __init__(self): wx.Frame.__init__(self, None, wx.NewId(), "Generated Test Frame") - sizer = wx.BoxSizer() - self.widget = widget_cls(self) # assumes need of parent. - sizer.Add(self.widget, 1, wx.EXPAND) + if issubclass(widget_cls, wx.Dialog): + dlg = widget_cls(self) + dlg.Show() # modeless + + else: + sizer = wx.BoxSizer() + self.widget = widget_cls(self) # assumes need of parent. + sizer.Add(self.widget, 1, wx.EXPAND) - self.SetSizer(sizer) - sizer.Layout() + self.SetSizer(sizer) + sizer.Layout() return BaseTestFrame diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 995d2fc2cf..148401aa41 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -46,8 +46,26 @@ def test_pass(self): def test_fail(self): self.testFailed("Deliberate test failure") +class ATCDialog(wx.Dialog, atc.TestWidget): + def __init__(self, parent): + wx.Dialog.__init__(self, parent) + atc.TestWidget.__init__(self) + + def test_pass(self): + self.testPassed() + + @unittest.expectedFailure + def test_faile(self): + self.testFailed() + + def test_intlw(self): + assert self in wx.GetTopLevelWindows(), "Dialog is not top level window" + self.testPassed() + atc_BasicTests = atc.createATC(ATCPanel) atc_FrameTests = atc.createATC(ATCFrame) +atc_DialogTests = atc.createATC(ATCDialog) + if __name__ == "__main__": unittest.main() From 1270bb28936369deb639b4efe786d11e2c2d41d2 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Wed, 22 Feb 2017 08:26:22 -0800 Subject: [PATCH 34/42] Do not impose showing the Frame in instances where the Frame is the custom widget --- unittests/atc.py | 4 +++- unittests/test_atc.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 1236da04bd..5a443c7f4b 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -250,7 +250,6 @@ class TestApp(wx.App): """ a generated App class """ def OnInit(self): self.frame = frame_cls() - self.frame.Show(True) return True return TestApp @@ -274,6 +273,7 @@ def __init__(self): if issubclass(widget_cls, wx.Dialog): dlg = widget_cls(self) + self.Show() # show first, so the dialog is on top dlg.Show() # modeless else: @@ -284,6 +284,8 @@ def __init__(self): self.SetSizer(sizer) sizer.Layout() + self.Show() + return BaseTestFrame def __CreateTestMethod(app_cls, case): diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 148401aa41..e08137ce59 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -48,14 +48,14 @@ def test_fail(self): class ATCDialog(wx.Dialog, atc.TestWidget): def __init__(self, parent): - wx.Dialog.__init__(self, parent) + wx.Dialog.__init__(self, parent, title = "ATCDialog") atc.TestWidget.__init__(self) def test_pass(self): self.testPassed() @unittest.expectedFailure - def test_faile(self): + def test_fail(self): self.testFailed() def test_intlw(self): From 51472c50e4a3bdb1aa1400db1195f5d64cec76c1 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Wed, 22 Feb 2017 08:26:22 -0800 Subject: [PATCH 35/42] Do not impose showing the Frame in instances where the Frame is the custom widget --- unittests/atc.py | 4 +++- unittests/test_atc.py | 4 ++-- unittests/test_frame.py | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 1236da04bd..5a443c7f4b 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -250,7 +250,6 @@ class TestApp(wx.App): """ a generated App class """ def OnInit(self): self.frame = frame_cls() - self.frame.Show(True) return True return TestApp @@ -274,6 +273,7 @@ def __init__(self): if issubclass(widget_cls, wx.Dialog): dlg = widget_cls(self) + self.Show() # show first, so the dialog is on top dlg.Show() # modeless else: @@ -284,6 +284,8 @@ def __init__(self): self.SetSizer(sizer) sizer.Layout() + self.Show() + return BaseTestFrame def __CreateTestMethod(app_cls, case): diff --git a/unittests/test_atc.py b/unittests/test_atc.py index 148401aa41..e08137ce59 100644 --- a/unittests/test_atc.py +++ b/unittests/test_atc.py @@ -48,14 +48,14 @@ def test_fail(self): class ATCDialog(wx.Dialog, atc.TestWidget): def __init__(self, parent): - wx.Dialog.__init__(self, parent) + wx.Dialog.__init__(self, parent, title = "ATCDialog") atc.TestWidget.__init__(self) def test_pass(self): self.testPassed() @unittest.expectedFailure - def test_faile(self): + def test_fail(self): self.testFailed() def test_intlw(self): diff --git a/unittests/test_frame.py b/unittests/test_frame.py index e43a311b21..c24b2cb2ef 100644 --- a/unittests/test_frame.py +++ b/unittests/test_frame.py @@ -68,6 +68,7 @@ def __init__(self): wx.Frame.__init__(self, None, wx.NewId(), "Frame Restore Test") atc.TestWidget.__init__(self) self.SetLabel("Frame Restore Test") + self.Show(True) def test_iconize_restore(self): # for some reason, gtk seems to need a bit more time after From f7ae86f94c5066cce68f127ba53b2058787b1dd7 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 23 Feb 2017 16:56:24 -0800 Subject: [PATCH 36/42] Added autoshow argument to allow test creators to disable automatically showing the Frame --- unittests/atc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 5a443c7f4b..1eed9e9fca 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -62,7 +62,7 @@ def method(*args, **kwargs): return method -def createATC(widget_cls): +def createATC(widget_cls, autoshow = True): """ Creates and returns a class that derives unittest.TestCase the TestCase is generated from widget IMPORTANT NOTE: In order for the returned class to be picked up by the default unittest TestDiscovery process @@ -78,7 +78,7 @@ def createATC(widget_cls): Args: widget: A widget *class* that derives from TestWidget. - + autoshow: (True) boolean value that indicates whether or not the top level window should be automatically shown Returns: unittest.TestCase derivation @@ -110,7 +110,7 @@ def test_fail(self): else: tlw = widget_cls - app = __CreateApp(tlw) + app = __CreateApp(tlw, autoshow) class ApplicationTestCase(unittest.TestCase): pass @@ -236,13 +236,14 @@ def __OnTest(self, evt): print("Testing: %s" % evt.case) testfunc() -def __CreateApp(frame_cls): +def __CreateApp(frame_cls, autoshow): """ Generates an app class that will create an instance of frame_cls on launch. This method is utilized inside atc and probably should not be used otherwise Args: frame_cls: Class that derives from wx.Frame + autoshow: boolean value that indicates whether or not the top level window should be automatically shown Returns: wx.App derived class """ @@ -250,6 +251,8 @@ class TestApp(wx.App): """ a generated App class """ def OnInit(self): self.frame = frame_cls() + if autoshow: + self.frame.Show() return True return TestApp From d532d0a59d9c9771c8ed2f03765315835212fce2 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 23 Feb 2017 22:56:00 -0800 Subject: [PATCH 37/42] updated generated frame naming to reflect the test widget's class name, similar to what occurs in wtc --- unittests/atc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/atc.py b/unittests/atc.py index 1eed9e9fca..ee82e8356c 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -272,7 +272,7 @@ def __CreateFrame(widget_cls): class BaseTestFrame(wx.Frame): """ A generated Frame class """ def __init__(self): - wx.Frame.__init__(self, None, wx.NewId(), "Generated Test Frame") + wx.Frame.__init__(self, None, wx.NewId(), "ATC: " + widget_cls.__name__) if issubclass(widget_cls, wx.Dialog): dlg = widget_cls(self) From c56cd5d3e4da7fef0a2b758911eccd408038e256 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Thu, 9 Mar 2017 12:14:51 -0800 Subject: [PATCH 38/42] Print traceback/stacktrace where applicable in the event of test failure. --- unittests/atc.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index ee82e8356c..357434277c 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -32,8 +32,10 @@ import functools import os +import six import sys import unittest +import traceback import wx import wx.lib.newevent @@ -54,8 +56,12 @@ def method(*args, **kwargs): try: func(*args, **kwargs) except Exception as e: - print("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) - # give full stack trace + six.print_("Unbound exception caught in test procedure:\n%s\n%s" % (e.__class__, str(e)), file = sys.stderr) + + # print the traceback for this exception + traceback.print_tb(sys.exc_info()[-1]) + + # close the app. wx.GetApp().exception = e for window in wx.GetTopLevelWindows(): window.Close() @@ -182,6 +188,18 @@ def testFailed(self, errmsg = "A test failed."): """ # do not rely on testCritical being applied to an above method + # print the stacktrace here, as there is no exception raised yet + stacktrace = traceback.extract_stack() + # find "test_func" or whatever the generated test function is called + # within stacktrace and exclude all rows before it. + for x in range(len(stacktrace)): + if "test_func" in str(stacktrace[x]): + stacktrace = stacktrace[x:-1] + break + + six.print_("".join(traceback.format_list(stacktrace)), file = sys.stderr) + six.print_("TestWidget.testFailed() called.", file = sys.stderr) + wx.GetApp().exception = TestError(errmsg) for window in wx.GetTopLevelWindows(): window.Close() @@ -217,7 +235,7 @@ def __OnWatchdog(self): """ Invoked with a test hass taken more than 5 minutes to complete """ - print("Test Timeout!!!") + six.print_("Test Timeout!!!") wx.GetApp().exception = RuntimeError("Watchdog timed out") for window in wx.GetTopLevelWindows(): window.Close() @@ -233,7 +251,7 @@ def __OnLnxStart(self, evt): def __OnTest(self, evt): testfunc = getattr(self, evt.case) - print("Testing: %s" % evt.case) + six.print_("Testing: %s" % evt.case) testfunc() def __CreateApp(frame_cls, autoshow): @@ -315,6 +333,9 @@ def test_func(obj): a.testcase = obj a.MainLoop() + if hasattr(a, "tb"): + traceback.print_tb(a.tb) + if hasattr(a, "exception"): raise a.exception From 23aff361b4f32091fca6130d604b1b241018f030 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Sat, 11 Mar 2017 20:43:03 -0800 Subject: [PATCH 39/42] Moved procedure to print stack trace to a new TestWidget method, adjusting framing accordingly. --- unittests/atc.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 357434277c..5c714ad422 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -28,7 +28,7 @@ # Add stack trace printouts upon TestDone(False) or TestCritical exception -__version__ = "0.0.2" +__version__ = "0.0.3" import functools import os @@ -188,18 +188,11 @@ def testFailed(self, errmsg = "A test failed."): """ # do not rely on testCritical being applied to an above method - # print the stacktrace here, as there is no exception raised yet - stacktrace = traceback.extract_stack() - # find "test_func" or whatever the generated test function is called - # within stacktrace and exclude all rows before it. - for x in range(len(stacktrace)): - if "test_func" in str(stacktrace[x]): - stacktrace = stacktrace[x:-1] - break - six.print_("".join(traceback.format_list(stacktrace)), file = sys.stderr) - six.print_("TestWidget.testFailed() called.", file = sys.stderr) - + # print stacktrace info (as no exception was raised at this point + # a stacktrace is used, not a traceback.) + self.__print_stacktrace() + wx.GetApp().exception = TestError(errmsg) for window in wx.GetTopLevelWindows(): window.Close() @@ -253,7 +246,24 @@ def __OnTest(self, evt): six.print_("Testing: %s" % evt.case) testfunc() + + def __print_stacktrace(self): + """ + Called during testFailed to print stack trace information. + """ + six.print_("Providing most recent stack trace information:\n", file = sys.stderr) + stacktrace = traceback.extract_stack() + # find "test_func" or whatever the generated test function is called + # within stacktrace and exclude all rows before it. + for x in range(len(stacktrace)): + if "test_func" in str(stacktrace[x]): + stacktrace = stacktrace[x:-2] # cut off call to this method + break # and call to extract_stack + + six.print_("".join(traceback.format_list(stacktrace)), file = sys.stderr) + six.print_("TestWidget.testFailed() called.", file = sys.stderr) + def __CreateApp(frame_cls, autoshow): """ Generates an app class that will create an instance of frame_cls on launch. From 901ee0565614184985382cbafe1fa68cea6643fa Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 22 May 2017 21:17:27 -0700 Subject: [PATCH 40/42] Add docstring describing ATC components --- unittests/atc.py | 56 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 5c714ad422..c01df0da0a 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -2,6 +2,51 @@ # Application Test Case # Allows testing wx features within the context of an application +""" +Application Test Case allows python's unittest framework to run full wx +applications within a test case with out any need to manually drive the main +eventloop. This reduces buginess within tested features that require a running +mainloop, such as event-based behavior. + +ATC accomplishes this by receiving a widget class and constructing a test case class +from the widget. When unittest invokes a test within this testcase the application +will be started and the test sequence will begin. + +The widget given to ATC will look like it is a test case itself, aside from a few +nuances. Which are as follows: +1) The widget MUST derive from the given class: TestWidget + This class provides the code necessary to automate tests and convey results. + Furthermore, anything that derives TestWidget must also derive from at least wx.EventHandler +2) Some methods will need to be decorated with 'testCritical' to perform as expected. + More on this later. +3) The ATC testcase class must be created and assigned to the global scope so unittest can detect it. + This is performed with createATC(test_widget_derivation) + +ATC requires some additional code in order to perform correctly. It needs: +1) to know which methods are critical. That is which ones are not allowed to have + an exception escape their frame. +2) When a test fails or passes + +First, a decorator method 'testCritical' is provided. this decorator will automatically fail +the current test if an an unhandled exception occurs within the decorated function. +Secondly, TestWidget provides the methods TestWidget.testPassed and TestWidget.TestFailed +to specify a test result. When these methods are called the application will exit and +results will be delivered to unittest + +Important notes: +The TestCase class can be accessed using TestWidget.getTestCase(), this is usefull for +utilizing standard testcase methods such as failUnless, assert____() and so on. +These TestCase methods will only work within testCritical methods. Otherwise +the exceptions raised by them will pass silently into the Python/wx sandwhich. + + +The following files contain ATC examples: +unittests/test_frame.py +unittest/test_atc.py + +~~ Samuel Dunn +""" + # Benefits: As stated, allows unit testing within a full App mainloop. # Allows developers to quickly construct a test case by developing a # single widget @@ -13,19 +58,10 @@ # Generated TestCase classes are unique, allowing plural per test # module -# Drawbacks: Generated test case class has to be applied to a __main__ module -# global scope member. Need to look into unittest.TestDiscorvery -# to correct this behavior -# Code is currently a bit sloppy, Revisions will be made before a -# formal PR to improve documentation and clean up the code. -# - # TODO: # Ensure full TestCase API is available within the app # automatically apply TestCritical decorator to test_ methods in widget -# Ensure TestCritical decorator plays nice with preserving other decorations -# such that decoration order does not matter -# Add stack trace printouts upon TestDone(False) or TestCritical exception +# Explore option of having the same application instance runn all test sequences. __version__ = "0.0.3" From f0b70827dad6392f213021823a2d2704922b12e6 Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 22 May 2017 21:17:52 -0700 Subject: [PATCH 41/42] remove deprecated comment block. --- unittests/atc.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index c01df0da0a..6e0b25e802 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -47,17 +47,6 @@ ~~ Samuel Dunn """ -# Benefits: As stated, allows unit testing within a full App mainloop. -# Allows developers to quickly construct a test case by developing a -# single widget -# TestCase class is generated from the widget provided to atc. The -# generated TestCase class mimics the widget to allow pytest -# to behave normally. This allows unsquashed testing (jamming -# plural tests into one runtime) -# Generated TestCase class should play nice with all unittest features -# Generated TestCase classes are unique, allowing plural per test -# module - # TODO: # Ensure full TestCase API is available within the app # automatically apply TestCritical decorator to test_ methods in widget From 5c9c98940d73fd726f55cb2f684dc6788dd4cfac Mon Sep 17 00:00:00 2001 From: Samuel Dunn Date: Mon, 22 May 2017 21:42:09 -0700 Subject: [PATCH 42/42] extraenous cleanup --- unittests/atc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unittests/atc.py b/unittests/atc.py index 6e0b25e802..a20282252c 100644 --- a/unittests/atc.py +++ b/unittests/atc.py @@ -145,11 +145,12 @@ def test_fail(self): class ApplicationTestCase(unittest.TestCase): pass - + methods = [meth for meth in dir(widget_cls) if (meth.startswith("test_") and callable(getattr(widget_cls, meth)))] for meth in methods: test_func = __CreateTestMethod(app, meth) + # ensure any other deocrated data is preserved: basemeth = getattr(widget_cls, meth) for attr in dir(basemeth): @@ -268,8 +269,6 @@ def __OnLnxStart(self, evt): @testCritical # automatically apply exception blocking to test_ methods.. indirectly def __OnTest(self, evt): testfunc = getattr(self, evt.case) - - six.print_("Testing: %s" % evt.case) testfunc() def __print_stacktrace(self):