//// TestCreator.cpp//// Created by Nissim Hadar on 2 Nov 2017.// Copyright 2013 High Fidelity, Inc.//// Distributed under the Apache License, Version 2.0.// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html//#include "TestCreator.h" #include <assert.h>#include <QtCore/QTextStream>#include <QDirIterator>#include <QHostInfo>#include <QImageReader>#include <QImageWriter> #include <quazip/quazip.h>#include <quazip/JlCompress.h> #include "Nitpick.h"extern Nitpick* nitpick; #include <math.h> TestCreator::TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _awsInterface(NULL) { _downloader = new Downloader(); _progressBar = progressBar; _checkBoxInteractiveMode = checkBoxInteractiveMode; _mismatchWindow.setModal(true); if (nitpick) { nitpick->setUserText(GIT_HUB_DEFAULT_USER); nitpick->setBranchText(GIT_HUB_DEFAULT_BRANCH); }} bool TestCreator::createTestResultsFolderPath(const QString& directory) { QDateTime now = QDateTime::currentDateTime(); _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT) + "(local)[" + QHostInfo::localHostName() + "]"; QDir testResultsFolder(_testResultsFolderPath); // Create a new test results folder return QDir().mkdir(_testResultsFolderPath);} QString TestCreator::zipAndDeleteTestResultsFolder() { QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (fileInfo.exists()) { QFile::remove(zippedResultsFileName); } QDir testResultsFolder(_testResultsFolderPath); JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath); testResultsFolder.removeRecursively(); //In all cases, for the next evaluation _testResultsFolderPath = ""; _failureIndex = 1; _successIndex = 1; return zippedResultsFileName;} int TestCreator::compareImageLists(const QString& gpuVendor) { _progressBar->setMinimum(0); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setValue(0); _progressBar->setVisible(true); // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. bool keepOn{ true }; int numberOfFailures{ 0 }; for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size QImage resultImage(_resultImagesFullFilenames[i]); QImage expectedImage(_expectedImagesFullFilenames[i]); double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical double worstTileValue; // in [-1.0 .. 1.0], where 1.0 means images are identical bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked() && !_isRunningInAutomaticTestRun); // similarityIndex is set to -100.0 to indicate images are not the same size if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); similarityIndex = -100.0; worstTileValue = 0.0; } else { _imageComparer.compareImages(resultImage, expectedImage); similarityIndex = _imageComparer.getSSIMValue(); worstTileValue = _imageComparer.getWorstTileValue(); } TestResult testResult = TestResult{ similarityIndex, worstTileValue, _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of result image _imageComparer.getSSIMResults() // results of SSIM algoritm }; _mismatchWindow.setTestResult(testResult); // Lower threshold for non-Nvidia GPUs double thresholdGlobal{ 0.9995 }; double thresholdLocal{ 0.6 }; if (gpuVendor != "Nvidia") { thresholdGlobal = 0.97; thresholdLocal = 0.2; } if (similarityIndex < thresholdGlobal || worstTileValue < thresholdLocal) { if (!isInteractiveMode) { ++numberOfFailures; appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true); } else { _mismatchWindow.exec(); switch (_mismatchWindow.getUserResponse()) { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: ++numberOfFailures; appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true); break; case USER_RESPONSE_ABORT: keepOn = false; break; default: assert(false); break; } } } else { appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), false); } _progressBar->setValue(i); } _progressBar->setVisible(false); return numberOfFailures;} int TestCreator::checkTextResults() { // Create lists of failed and passed tests QStringList nameFilterFailed; nameFilterFailed << "*.failed.txt"; QStringList testsFailed = QDir(_snapshotDirectory).entryList(nameFilterFailed, QDir::Files, QDir::Name); QStringList nameFilterPassed; nameFilterPassed << "*.passed.txt"; QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); // Add results to TestCreator Results folder foreach(QString currentFilename, testsFailed) { appendTestResultsToFile(currentFilename, true); } foreach(QString currentFilename, testsPassed) { appendTestResultsToFile(currentFilename, false); } return testsFailed.length();} void TestCreator::appendTestResultsToFile(const TestResult& testResult, const QPixmap& comparisonImage, const QPixmap& ssimResultsImage, bool hasFailed) { // Critical error if Test Results folder does not exist if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } // There are separate subfolders for failures and passes QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); ++_failureIndex; } else { resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); ++_successIndex; } if (!QDir().mkdir(resultFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + resultFolderPath); exit(-1); } QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME); exit(-1); } // Create text file describing the failure QTextStream stream(&descriptionFile); stream << "TestCreator in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << Qt::endl; // remove trailing '/' stream << "Expected image was " << testResult._expectedImageFilename << Qt::endl; stream << "Actual image was " << testResult._actualImageFilename << Qt::endl; stream << "Similarity index was " << testResult._errorGlobal << Qt::endl; stream << "Worst tile was " << testResult._errorLocal << Qt::endl; descriptionFile.close(); // Copy expected and actual images, and save the difference image QString sourceFile; QString destinationFile; sourceFile = testResult._pathname + testResult._expectedImageFilename; destinationFile = resultFolderPath + "/" + "Expected Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } sourceFile = testResult._pathname + testResult._actualImageFilename; destinationFile = resultFolderPath + "/" + "Actual Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); // Save the SSIM results image ssimResultsImage.save(resultFolderPath + "/" + "SSIM Image.png");} void::TestCreator::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { // The test name includes everything until the penultimate period QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; ++_failureIndex; } else { resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; ++_successIndex; } if (!QDir().mkdir(resultFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + resultFolderPath); exit(-1); } QString source = _snapshotDirectory + "/" + testResultFilename; QString destination = resultFolderPath + "/Result.txt"; if (!QFile::copy(source, destination)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); exit(-1); }} void TestCreator::startTestsEvaluation( QComboBox *gpuVendor, const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, const QString& branchFromCommandLine, const QString& userFromCommandLine) { _isRunningFromCommandLine = isRunningFromCommandLine; _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun; if (snapshotDirectory.isNull()) { // Get list of PNG images in folder, sorted by name QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_snapshotDirectory == "") { _snapshotDirectory = previousSelection; return; } } else { _snapshotDirectory = snapshotDirectory; _exitWhenComplete = (isRunningFromCommandLine && !isRunningInAutomaticTestRun); } // Quit if test results folder could not be created if (!createTestResultsFolderPath(_snapshotDirectory)) { return; } // Create two lists. The first is the test results, the second is the expected images // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); QStringList expectedImagesURLs; _resultImagesFullFilenames.clear(); _expectedImagesFilenames.clear(); _expectedImagesFullFilenames.clear(); QString branch = (branchFromCommandLine.isNull()) ? nitpick->getSelectedBranch() : branchFromCommandLine; QString user = (userFromCommandLine.isNull()) ? nitpick->getSelectedUser() : userFromCommandLine; foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { _resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); // Images are stored on GitHub as ExpectedImage_ddddd.png or ExpectedImage_some_metadata_ddddd.png // Extract the part of the filename after "ExpectedImage_" and excluding the file extension QString expectedImageFilenameTail = currentFilename.left(currentFilename.lastIndexOf(".")); int expectedImageStart = expectedImageFilenameTail.lastIndexOf(".") + 1; expectedImageFilenameTail.remove(0, expectedImageStart); QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); expectedImagesURLs << imageURLString; // The image retrieved from GitHub needs a unique name QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png"); _expectedImagesFilenames << expectedImageFilename; _expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename; } } _downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); finishTestsEvaluation(gpuVendor->currentText());} void TestCreator::finishTestsEvaluation(const QString& gpuVendor) { // First - compare the pairs of images int numberOfFailures = compareImageLists(gpuVendor); // Next - check text results numberOfFailures += checkTextResults(); if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); } else if (numberOfFailures == 1) { QMessageBox::information(0, "Failure", "One image is not as expected"); } else { QMessageBox::information(0, "Failure", QString::number(numberOfFailures) + " images are not as expected"); } } QString zippedFolderName = zipAndDeleteTestResultsFolder(); if (_exitWhenComplete) { exit(0); } if (_isRunningInAutomaticTestRun) { nitpick->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); }} bool TestCreator::isAValidDirectory(const QString& pathname) { // Only process directories QDir dir(pathname); if (!dir.exists()) { return false; } // Ignore '.', '..' directories if (pathname[pathname.length() - 1] == '.') { return false; } return true;} QString TestCreator::extractPathFromTestsDown(const QString& fullPath) { // `fullPath` includes the full path to the test. We need the portion below (and including) `tests` QStringList pathParts = fullPath.split('/'); int i{ 0 }; while (i < pathParts.length() && pathParts[i] != "tests") { ++i; } if (i == pathParts.length()) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname"); exit(-1); } QString partialPath; for (int j = i; j < pathParts.length(); ++j) { partialPath += "/" + pathParts[j]; } return partialPath;} void TestCreator::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << Qt::endl;} void TestCreator::createTests(const QString& clientProfile) { // Rename files sequentially, as ExpectedResult_00000.png, ExpectedResult_00001.png and so on // Any existing expected result images will be deleted QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_snapshotDirectory == "") { _snapshotDirectory = previousSelection; return; } previousSelection = _testsRootDirectory; parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } if (!createAllFilesSetup()) { return; } // If user canceled then restore previous selection and return if (_testsRootDirectory == "") { _testsRootDirectory = previousSelection; return; } QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } // Path to test is extracted from the file name // Example: // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg // path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd // // Note: we don't use the first part and the last 2 parts of the filename at this stage // QStringList pathParts = currentFilename.split("."); QString fullNewFileName = _testsRootDirectory; for (int j = 1; j < pathParts.size() - 2; ++j) { fullNewFileName += "/" + pathParts[j]; } // The image _index is the penultimate component of the path parts (the last being the file extension) QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; fullNewFileName += "/" + newFilename; try { if (QFile::exists(fullNewFileName)) { QFile::remove(fullNewFileName); } QFile::copy(fullCurrentFilename, fullNewFileName); } catch (...) { QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n"); exit(-1); } ++i; } } QMessageBox::information(0, "Success", "Test images have been created");} namespace TestProfile {Lines 1–500 of 1292. Load more
// // TestCreator.cpp // // Created by Nissim Hadar on 2 Nov 2017. // Copyright 2013 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "TestCreator.h" #include <assert.h> #include <QtCore/QTextStream> #include <QDirIterator> #include <QHostInfo> #include <QImageReader> #include <QImageWriter> #include <quazip/quazip.h> #include <quazip/JlCompress.h> #include "Nitpick.h" extern Nitpick* nitpick; #include <math.h> TestCreator::TestCreator(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode) : _awsInterface(NULL) { _downloader = new Downloader(); _progressBar = progressBar; _checkBoxInteractiveMode = checkBoxInteractiveMode; _mismatchWindow.setModal(true); if (nitpick) { nitpick->setUserText(GIT_HUB_DEFAULT_USER); nitpick->setBranchText(GIT_HUB_DEFAULT_BRANCH); } } bool TestCreator::createTestResultsFolderPath(const QString& directory) { QDateTime now = QDateTime::currentDateTime(); _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT) + "(local)[" + QHostInfo::localHostName() + "]"; QDir testResultsFolder(_testResultsFolderPath); // Create a new test results folder return QDir().mkdir(_testResultsFolderPath); } QString TestCreator::zipAndDeleteTestResultsFolder() { QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (fileInfo.exists()) { QFile::remove(zippedResultsFileName); } QDir testResultsFolder(_testResultsFolderPath); JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath); testResultsFolder.removeRecursively(); //In all cases, for the next evaluation _testResultsFolderPath = ""; _failureIndex = 1; _successIndex = 1; return zippedResultsFileName; } int TestCreator::compareImageLists(const QString& gpuVendor) { _progressBar->setMinimum(0); _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); _progressBar->setValue(0); _progressBar->setVisible(true); // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. bool keepOn{ true }; int numberOfFailures{ 0 }; for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size QImage resultImage(_resultImagesFullFilenames[i]); QImage expectedImage(_expectedImagesFullFilenames[i]); double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical double worstTileValue; // in [-1.0 .. 1.0], where 1.0 means images are identical bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked() && !_isRunningInAutomaticTestRun); // similarityIndex is set to -100.0 to indicate images are not the same size if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); similarityIndex = -100.0; worstTileValue = 0.0; } else { _imageComparer.compareImages(resultImage, expectedImage); similarityIndex = _imageComparer.getSSIMValue(); worstTileValue = _imageComparer.getWorstTileValue(); } TestResult testResult = TestResult{ similarityIndex, worstTileValue, _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of result image _imageComparer.getSSIMResults() // results of SSIM algoritm }; _mismatchWindow.setTestResult(testResult); // Lower threshold for non-Nvidia GPUs double thresholdGlobal{ 0.9995 }; double thresholdLocal{ 0.6 }; if (gpuVendor != "Nvidia") { thresholdGlobal = 0.97; thresholdLocal = 0.2; } if (similarityIndex < thresholdGlobal || worstTileValue < thresholdLocal) { if (!isInteractiveMode) { ++numberOfFailures; appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true); } else { _mismatchWindow.exec(); switch (_mismatchWindow.getUserResponse()) { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: ++numberOfFailures; appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true); break; case USER_RESPONSE_ABORT: keepOn = false; break; default: assert(false); break; } } } else { appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), false); } _progressBar->setValue(i); } _progressBar->setVisible(false); return numberOfFailures; } int TestCreator::checkTextResults() { // Create lists of failed and passed tests QStringList nameFilterFailed; nameFilterFailed << "*.failed.txt"; QStringList testsFailed = QDir(_snapshotDirectory).entryList(nameFilterFailed, QDir::Files, QDir::Name); QStringList nameFilterPassed; nameFilterPassed << "*.passed.txt"; QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); // Add results to TestCreator Results folder foreach(QString currentFilename, testsFailed) { appendTestResultsToFile(currentFilename, true); } foreach(QString currentFilename, testsPassed) { appendTestResultsToFile(currentFilename, false); } return testsFailed.length(); } void TestCreator::appendTestResultsToFile(const TestResult& testResult, const QPixmap& comparisonImage, const QPixmap& ssimResultsImage, bool hasFailed) { // Critical error if Test Results folder does not exist if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } // There are separate subfolders for failures and passes QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); ++_failureIndex; } else { resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); ++_successIndex; } if (!QDir().mkdir(resultFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + resultFolderPath); exit(-1); } QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME); exit(-1); } // Create text file describing the failure QTextStream stream(&descriptionFile); stream << "TestCreator in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << Qt::endl; // remove trailing '/' stream << "Expected image was " << testResult._expectedImageFilename << Qt::endl; stream << "Actual image was " << testResult._actualImageFilename << Qt::endl; stream << "Similarity index was " << testResult._errorGlobal << Qt::endl; stream << "Worst tile was " << testResult._errorLocal << Qt::endl; descriptionFile.close(); // Copy expected and actual images, and save the difference image QString sourceFile; QString destinationFile; sourceFile = testResult._pathname + testResult._expectedImageFilename; destinationFile = resultFolderPath + "/" + "Expected Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } sourceFile = testResult._pathname + testResult._actualImageFilename; destinationFile = resultFolderPath + "/" + "Actual Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); // Save the SSIM results image ssimResultsImage.save(resultFolderPath + "/" + "SSIM Image.png"); } void::TestCreator::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { // The test name includes everything until the penultimate period QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; ++_failureIndex; } else { resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; ++_successIndex; } if (!QDir().mkdir(resultFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + resultFolderPath); exit(-1); } QString source = _snapshotDirectory + "/" + testResultFilename; QString destination = resultFolderPath + "/Result.txt"; if (!QFile::copy(source, destination)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); exit(-1); } } void TestCreator::startTestsEvaluation( QComboBox *gpuVendor, const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, const QString& branchFromCommandLine, const QString& userFromCommandLine ) { _isRunningFromCommandLine = isRunningFromCommandLine; _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun; if (snapshotDirectory.isNull()) { // Get list of PNG images in folder, sorted by name QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_snapshotDirectory == "") { _snapshotDirectory = previousSelection; return; } } else { _snapshotDirectory = snapshotDirectory; _exitWhenComplete = (isRunningFromCommandLine && !isRunningInAutomaticTestRun); } // Quit if test results folder could not be created if (!createTestResultsFolderPath(_snapshotDirectory)) { return; } // Create two lists. The first is the test results, the second is the expected images // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); QStringList expectedImagesURLs; _resultImagesFullFilenames.clear(); _expectedImagesFilenames.clear(); _expectedImagesFullFilenames.clear(); QString branch = (branchFromCommandLine.isNull()) ? nitpick->getSelectedBranch() : branchFromCommandLine; QString user = (userFromCommandLine.isNull()) ? nitpick->getSelectedUser() : userFromCommandLine; foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { _resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); // Images are stored on GitHub as ExpectedImage_ddddd.png or ExpectedImage_some_metadata_ddddd.png // Extract the part of the filename after "ExpectedImage_" and excluding the file extension QString expectedImageFilenameTail = currentFilename.left(currentFilename.lastIndexOf(".")); int expectedImageStart = expectedImageFilenameTail.lastIndexOf(".") + 1; expectedImageFilenameTail.remove(0, expectedImageStart); QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); expectedImagesURLs << imageURLString; // The image retrieved from GitHub needs a unique name QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png"); _expectedImagesFilenames << expectedImageFilename; _expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename; } } _downloader->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); finishTestsEvaluation(gpuVendor->currentText()); } void TestCreator::finishTestsEvaluation(const QString& gpuVendor) { // First - compare the pairs of images int numberOfFailures = compareImageLists(gpuVendor); // Next - check text results numberOfFailures += checkTextResults(); if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); } else if (numberOfFailures == 1) { QMessageBox::information(0, "Failure", "One image is not as expected"); } else { QMessageBox::information(0, "Failure", QString::number(numberOfFailures) + " images are not as expected"); } } QString zippedFolderName = zipAndDeleteTestResultsFolder(); if (_exitWhenComplete) { exit(0); } if (_isRunningInAutomaticTestRun) { nitpick->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures); } } bool TestCreator::isAValidDirectory(const QString& pathname) { // Only process directories QDir dir(pathname); if (!dir.exists()) { return false; } // Ignore '.', '..' directories if (pathname[pathname.length() - 1] == '.') { return false; } return true; } QString TestCreator::extractPathFromTestsDown(const QString& fullPath) { // `fullPath` includes the full path to the test. We need the portion below (and including) `tests` QStringList pathParts = fullPath.split('/'); int i{ 0 }; while (i < pathParts.length() && pathParts[i] != "tests") { ++i; } if (i == pathParts.length()) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Bad testPathname"); exit(-1); } QString partialPath; for (int j = i; j < pathParts.length(); ++j) { partialPath += "/" + pathParts[j]; } return partialPath; } void TestCreator::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << Qt::endl; } void TestCreator::createTests(const QString& clientProfile) { // Rename files sequentially, as ExpectedResult_00000.png, ExpectedResult_00001.png and so on // Any existing expected result images will be deleted QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return if (_snapshotDirectory == "") { _snapshotDirectory = previousSelection; return; } previousSelection = _testsRootDirectory; parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } if (!createAllFilesSetup()) { return; } // If user canceled then restore previous selection and return if (_testsRootDirectory == "") { _testsRootDirectory = previousSelection; return; } QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } // Path to test is extracted from the file name // Example: // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg // path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd // // Note: we don't use the first part and the last 2 parts of the filename at this stage // QStringList pathParts = currentFilename.split("."); QString fullNewFileName = _testsRootDirectory; for (int j = 1; j < pathParts.size() - 2; ++j) { fullNewFileName += "/" + pathParts[j]; } // The image _index is the penultimate component of the path parts (the last being the file extension) QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; fullNewFileName += "/" + newFilename; try { if (QFile::exists(fullNewFileName)) { QFile::remove(fullNewFileName); } QFile::copy(fullCurrentFilename, fullNewFileName); } catch (...) { QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n"); exit(-1); } ++i; } } QMessageBox::information(0, "Success", "Test images have been created"); } namespace TestProfile {