Workflow Test Runner in Azure Devops

#
# Python 3 is required.
#
# python3 workflowtests/scripts/workflowtestrunner.py
#
# The following python packages are required.
#
# pip install junitparser
# pip install junit2html
# pip install requests
# pip install pytest

# -*- coding: utf-8 -*-
import sys
import os
import shutil
import requests, zipfile, io
from datetime import datetime
from junitparser import JUnitXml

# Make sure the basebuild is available.
def download_base_build():
    r = requests.get(
        "https://delftfewsbasebuilds.blob.core.windows.net/delft-fews-base-builds/fews-development-202201-109669-bin.zip")
    z = zipfile.ZipFile(io.BytesIO(r.content))
    z.extractall("bin")


def absolute_file_paths(directory):
    path = os.path.abspath(directory)
    return [entry.path for entry in os.scandir(path) if entry.is_file()]


def merge_xml_reports(reportDir):
    junit_report_dir = f"{reportDir}/junit"
    report = ""
    junit_files = absolute_file_paths(junit_report_dir)

    for file in junit_files:
        if file.endswith(".xml"):
            xml = JUnitXml.fromfile(file)
            if report == "":
                report = xml
            else:
                report = report + xml

    report_xml_file = f"{reportDir}/junit/report-all.xml"
    report.write(report_xml_file, True)
    return report_xml_file


def main(argv):
    working_dir = os.getcwd()
    if not os.path.isdir(f"{working_dir}/bin"):
        download_base_build()
    report_dir = f"{working_dir}/report"
    report_html_file = f"{report_dir}/report.html"
    shutil.rmtree(report_dir, ignore_errors=True)
    workflow_test_files = f"{working_dir}/workflowtests/configuration"
    python_test_files = f"{working_dir}/workflowtests/scripts"

    for file in sorted(os.listdir(workflow_test_files)):
        filename = os.fsdecode(file)
        if filename.endswith(".xml"):
            print(f"Start workflow test for file: {filename}")
            workflow_tests_command = f"{working_dir}/bin/windows/Delft-FEWSc.exe -Dregion.home={working_dir}/region_home -DautoRollingBarrel=false -Xmx20484m -DoldPID=13096_1555070573358 -Djava.locale.providers=SPI,JRE -Dstart.time=1630454400000 -Djava.library.path={working_dir}/bin/windows -Wvm.location={working_dir}/bin/windows/jre/bin/server/jvm.dll -Wclasspath.1={working_dir}/region_home/patch.jar -Wclasspath.2={working_dir}/bin/*.jar -Wmain.class=nl.wldelft.fews.system.workflowtestrun.WorkflowTestRun -Warg.1={working_dir}/region_home -Warg.2={working_dir}/workflowtests/configuration/{filename}"
            print("%s", workflow_tests_command)
            # Run the workflow tests.
            os.system(workflow_tests_command)

            # The following is an option part.
            # Enable to run a python test directly after the workflow test.
            # In case the workflow test is called: test1.xml, the python test should be named test1.py in the scripts folder.
            python_test_case = filename.removesuffix('xml')
            python_test_file = f'{python_test_files}/{python_test_case}py'
            print(f'Check if there is a test python test script to run: {python_test_file}')
            if  os.path.isfile(python_test_file):
                # In case some tests need to be run after the workflow test runner.
                py_test_command = f"pytest --junitxml={report_dir}/junit/{python_test_case}py.xml {python_test_file}"
                # run the python test.
                os.system(py_test_command)

    # Merge all xml reports.
    report_xml_file = merge_xml_reports(report_dir)
    os.system(f"junit2html {report_xml_file} {report_html_file}")

# ------------------------------------------------------------------------------
if __name__ == "__main__":
    start_time = datetime.now()
    print('Start Workflow Test Runner: %s\n' % start_time.strftime("%Y-%m-%d %H:%M:%S"))
    main(sys.argv[0:])
    print('End  : %s' % datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    print('Finished')  

# Build Pipeline to test Delft-FEWS configurations using the Workflow Test Runner
# Any commit in te configuration, will trigger the pipeline.

# Only trigger the build on commits in the region_home folder.
trigger:
  branches:
    include:
    - main
  paths:
    include:
    - region_home

pool:
  vmImage: windows-2019

steps:

- checkout: self
  path: fews
# Checkout the configuration repo to the fews folder.
- checkout: git://MDBADelftFewsConfigurationPipeline/MDBADelftFewsConfigurationPipeline
  path: testfiles

- task: UsePythonVersion@0
  inputs:
    versionSpec: '3.10'

- script: |
    echo %cd%
    pip install pytest pytest-azurepipelines
    pip install junitparser
    pip install junit2html
    pip install requests
    cd ../fews
    echo %cd%
    python workflowtests/scripts/workflowtestrunner.py
  displayName: 'Run Delft-FEWS Workflow Test Runner.'    
    
# Publish test results to Azure Pipelines
- task: PublishTestResults@2
  inputs:
    testResultsFormat: 'JUnit'
    testResultsFiles: '../fews/report/junit/report-all.xml' 
    failTaskOnFailedTests: true # Optional
    #testRunTitle: # Optional
    #buildPlatform: # Optional
    #buildConfiguration: # Optional
    publishRunAttachments: true
  displayName: 'Publish Delft-FEWS Workflow Test Runner results.'

- task: ArchiveFiles@2
  inputs:
    rootFolderOrFile: '../fews/region_home/Config'
    includeRootFolder: true
    archiveType: 'zip'
    archiveFile: '$(Build.ArtifactStagingDirectory)/Config.zip'
    replaceExistingArchive: true
  displayName: 'Zip Delft-FEWS config.'

- task: PublishBuildArtifacts@1
  displayName: 'Publish Delft-FEWS Config.zip'