cumulative_coverage.py

# This script is licensed under BSD License, see file LICENSE.txt.
#
# (c) TASKING Germany GmbH, 2023
 
import isystem.connect as ic
import isystem.itest as it
 
winidea_id = ''
 
 
try:
    import pylab as pl
    is_pylab_available = True
except Exception as ex:
    is_pylab_available = False
 
 
class Monitor:
    def __init__(self, no_of_sub_tasks):
        self.no_of_sub_tasks = no_of_sub_tasks
        self.test_count = 0
 
    def isCanceled(self):
        pass
 
    def subTask(self, task_desc):
        print(task_desc)
 
    def worked(self, count):
        self.test_count += count
        print('#' * self.test_count + '-' * (self.no_of_sub_tasks - self.test_count))
 
 
def init_target() -> ic.ConnectionMgr:
    cmgr = ic.ConnectionMgr()
 
    cmgr.connect(ic.CConnectionConfig().instanceId(winidea_id))
 
    debug_ctrl = ic.CDebugFacade(cmgr)
    session_ctrl = ic.CSessionCtrl(cmgr)
 
 
    session_ctrl.begin_reset()
 
    debug_ctrl.runUntilFunction('main')
    debug_ctrl.waitUntilStopped()
 
    return cmgr
 
 
def run_test(connection_mgr: ic.ConnectionMgr) -> ic.CTestReportContainer:
    test_case = it.PTestCase(connection_mgr)
 
    test_case.reset()  # just in case there's something left from previous run
 
    # Test specification contains everything needed for code coverage
    # measurements. It would also be possible to remove the first test
    # specification, which configures coverage and replace it with
    # isystem.connect calls. See 'cumulativeModuleCoverage.py' for example.
    test_spec = ic.CTestSpecification.parseTestSpec(
        """
    # the first test specification configures coverage
    func: [fibonacci, [0], rv]
    coverage:
        runMode: start    # coverage should be configured
        document: fibonacci.trd
        openMode: w        # write mode erases any previous coverage data
        isDecisionCoverage: true
        statistics:
        - func: fibonacci
    tests:
      # abstract base specification follows, which defines coverage for all
      # derived tests. 
      - func: [fibonacci, [0], rv]
        run: no  # this is an abstract base test
        coverage:
            runMode: resume    # coverage should be accumulated
            document: fibonacci.trd
            openMode: u        # 'update' mode keeps previous coverage data
            statistics:
            - func: fibonacci
        tests: # derived tests execute the function for several parameters to get full coverage
            - params: [0]
              expect: [rv == 0]
          
            - params: [1]
              expect: [rv == 1]
          
            - params: [2]
              expect: [rv == 1]
          
            - params: [3]
              expect: [rv == 2]
          
            - params: [7]
              expect: [rv == 13 @@ d]
              coverage:  # the last test should check for coverage values
                  runMode: resume
                  document: fibonacci.trd
                  openMode: u
                  isDecisionCoverage: true
                  statistics:
                  - func: fibonacci
                    code: 100
                    branches: 100
                    taken: 100
                    notTaken: 100
                    both: 100
    """)
 
    monitor = Monitor(test_spec.getNoOfTests(True))
 
    print('Running tests to achieve full coverage...')
    test_case.runTests(test_spec, None, monitor)
 
    test_results = test_case.getTestResultsContainer()
    return test_results
 
 
def save_test_Results(test_results: ic.CTestReportContainer, resultFileName: str):
    error_file_stream = ic.CFileStream(resultFileName)
    err_emitter = ic.EmitterFactory.createCSVEmitter(error_file_stream, ",", True)
    test_results.resetTestResultIterator()
 
    while test_results.hasNextTestResult():
        result = test_results.nextTestResult()
        # serialize result to file
        err_emitter.startDocument(False)
        result.serialize(err_emitter, None)
        err_emitter.endDocument(False)
 
        # serialize result to string for printing to stdout
        err_stream = ic.CStringStream()
        emitter = ic.EmitterFactory.createYamlEmitter(err_stream)
 
        emitter.startStream()
        emitter.startDocument(False)
        result.serializeErrorsOnly(emitter, None)
        emitter.endDocument(False)
        emitter.endStream()
 
        print(err_stream.getString())
 
    err_emitter.endStream()
    error_file_stream.close()
 
 
def plot_coverage(test_results: ic.CTestReportContainer, is_plot_all=False):
    """
    Draws a plot showing the amount of coverage in each test.
    This function requires 'matplotlib' and 'numpy' Python packages to be
    installed.
    """
    bytes_executed = []
    lines_executed = []
    branches_executed = []
    branches_taken = []
    branches_not_taken = []
    branches_both = []
 
    test_results.resetTestResultIterator()
    while test_results.hasNextTestResult():
        result = test_results.nextTestResult()
        cvrg_result = ic.StrCoverageTestResultsMap()
        result.getCoverageResults(cvrg_result)
        for function in cvrg_result:
            stats = cvrg_result[function].getMeasuredCoverage(True)
 
            bytes_executed.append(stats.getBytesExecuted())
            lines_executed.append(stats.getSourceLinesExecuted())
            branches_executed.append(stats.getBranchExecuted())
            branches_taken.append(stats.getBranchTaken())
            branches_not_taken.append(stats.getBranchNotTaken())
            branches_both.append(stats.getBranchBoth())
 
    x = range(len(bytes_executed))
    pl.plot(x, bytes_executed, '.-', label='bytes executed')
    pl.plot(x, branches_both, '.-', label='branches both')
    if is_plot_all:
        pl.plot(x, lines_executed, 'o', label='lines executed')
        pl.plot(x, branches_taken, '--', label='branches taken')
        pl.plot(x, branches_not_taken, '--', label='branches not taken')
        pl.plot(x, branches_executed, '--', label='branches executed')
 
    pl.axis([0, 5, 0, 110])
 
    pl.xlabel('test number')
    pl.ylabel('%')
    pl.title("Coverage of function under test")
    pl.legend()
    pl.grid(True)
    pl.show()
 
 
def main():
    cmgr = init_target()
    test_results = run_test(cmgr)
 
    result_file_name = 'errorsReport.csv'
    print('Tests finished, saving results to file: ', result_file_name)
    save_test_Results(test_results, result_file_name)
 
    if is_pylab_available:
        print('Showing results in graph (matplotlib and numpy packages must be installed!).')
        plot_coverage(test_results)
    else:
        print("Can't plot coverage results, pylab module is not available.")
 
 
if __name__ == '__main__':
    main()