SDL/visualtest/src/testharness.c

534 lines
15 KiB
C

/* See LICENSE.txt for the full license governing this code. */
/**
* \file testharness.c
*
* Source file for the test harness.
*/
#include <stdlib.h>
#include <SDL_test.h>
#include <SDL.h>
#include <SDL_assert.h>
#include "SDL_visualtest_harness_argparser.h"
#include "SDL_visualtest_process.h"
#include "SDL_visualtest_variators.h"
#include "SDL_visualtest_screenshot.h"
#include "SDL_visualtest_mischelper.h"
#if defined(__WIN32__) && !defined(__CYGWIN__)
#include <direct.h>
#elif defined(__WIN32__) && defined(__CYGWIN__)
#include <signal.h>
#elif defined(__LINUX__)
#include <sys/stat.h>
#include <sys/types.h>
#include <signal.h>
#else
#error "Unsupported platform"
#endif
/** Code for the user event triggered when a new action is to be executed */
#define ACTION_TIMER_EVENT 0
/** Code for the user event triggered when the maximum timeout is reached */
#define KILL_TIMER_EVENT 1
/** FPS value used for delays in the action loop */
#define ACTION_LOOP_FPS 10
/** Value returned by RunSUTAndTest() when the test has passed */
#define TEST_PASSED 1
/** Value returned by RunSUTAndTest() when the test has failed */
#define TEST_FAILED 0
/** Value returned by RunSUTAndTest() on a fatal error */
#define TEST_ERROR -1
static SDL_ProcessInfo pinfo;
static SDL_ProcessExitStatus sut_exitstatus;
static SDLVisualTest_HarnessState state;
static SDLVisualTest_Variator variator;
static SDLVisualTest_ActionNode* current; /* the current action being performed */
static SDL_TimerID action_timer, kill_timer;
/* returns a char* to be passed as the format argument of a printf-style function. */
static char*
usage()
{
return "Usage: \n%s --sutapp xyz"
" [--sutargs abc | --parameter-config xyz.parameters"
" [--variator exhaustive|random]"
" [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
" [--action-config xyz.actions]"
" [--output-dir /path/to/output]"
" [--verify-dir /path/to/verify]"
" or --config app.config";
}
/* register Ctrl+C handlers */
#if defined(__LINUX__) || defined(__CYGWIN__)
static void
CtrlCHandlerCallback(int signum)
{
SDL_Event event;
SDLTest_Log("Ctrl+C received");
event.type = SDL_QUIT;
SDL_PushEvent(&event);
}
#endif
static Uint32
ActionTimerCallback(Uint32 interval, void* param)
{
SDL_Event event;
SDL_UserEvent userevent;
Uint32 next_action_time;
/* push an event to handle the action */
userevent.type = SDL_USEREVENT;
userevent.code = ACTION_TIMER_EVENT;
userevent.data1 = &current->action;
userevent.data2 = NULL;
event.type = SDL_USEREVENT;
event.user = userevent;
SDL_PushEvent(&event);
/* calculate the new interval and return it */
if(current->next)
next_action_time = current->next->action.time - current->action.time;
else
{
next_action_time = 0;
action_timer = 0;
}
current = current->next;
return next_action_time;
}
static Uint32
KillTimerCallback(Uint32 interval, void* param)
{
SDL_Event event;
SDL_UserEvent userevent;
userevent.type = SDL_USEREVENT;
userevent.code = KILL_TIMER_EVENT;
userevent.data1 = NULL;
userevent.data2 = NULL;
event.type = SDL_USEREVENT;
event.user = userevent;
SDL_PushEvent(&event);
kill_timer = 0;
return 0;
}
static int
ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
{
if(!action || !sut_running)
return TEST_ERROR;
switch(action->type)
{
case SDL_ACTION_KILL:
SDLTest_Log("Action: Kill SUT");
if(SDL_IsProcessRunning(&pinfo) == 1 &&
!SDL_KillProcess(&pinfo, &sut_exitstatus))
{
SDLTest_LogError("SDL_KillProcess() failed");
return TEST_ERROR;
}
*sut_running = 0;
break;
case SDL_ACTION_QUIT:
SDLTest_Log("Action: Quit SUT");
if(SDL_IsProcessRunning(&pinfo) == 1 &&
!SDL_QuitProcess(&pinfo, &sut_exitstatus))
{
SDLTest_LogError("SDL_QuitProcess() failed");
return TEST_FAILED;
}
*sut_running = 0;
break;
case SDL_ACTION_LAUNCH:
{
char* path;
char* args;
SDL_ProcessInfo action_process;
SDL_ProcessExitStatus ps;
path = action->extra.process.path;
args = action->extra.process.args;
if(args)
{
SDLTest_Log("Action: Launch process: %s with arguments: %s",
path, args);
}
else
SDLTest_Log("Action: Launch process: %s", path);
if(!SDL_LaunchProcess(path, args, &action_process))
{
SDLTest_LogError("SDL_LaunchProcess() failed");
return TEST_ERROR;
}
/* small delay so that the process can do its job */
SDL_Delay(1000);
if(SDL_IsProcessRunning(&action_process) > 0)
{
SDLTest_LogError("Process %s took too long too complete."
" Force killing...", action->extra);
if(!SDL_KillProcess(&action_process, &ps))
{
SDLTest_LogError("SDL_KillProcess() failed");
return TEST_ERROR;
}
}
}
break;
case SDL_ACTION_SCREENSHOT:
{
char path[MAX_PATH_LEN], hash[33];
SDLTest_Log("Action: Take screenshot");
/* can't take a screenshot if the SUT isn't running */
if(SDL_IsProcessRunning(&pinfo) != 1)
{
SDLTest_LogError("SUT has quit.");
*sut_running = 0;
return TEST_FAILED;
}
/* file name for the screenshot image */
SDLVisualTest_HashString(args, hash);
SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
{
SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
return TEST_ERROR;
}
}
break;
case SDL_ACTION_VERIFY:
{
int ret;
SDLTest_Log("Action: Verify screenshot");
ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
state.verify_dir);
if(ret == -1)
{
SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
return TEST_ERROR;
}
else if(ret == 0)
{
SDLTest_Log("Verification failed: Images were not equal.");
return TEST_FAILED;
}
else if(ret == 1)
SDLTest_Log("Verification successful.");
else
{
SDLTest_Log("Verfication skipped.");
return TEST_FAILED;
}
}
break;
default:
SDLTest_LogError("Invalid action type");
return TEST_ERROR;
break;
}
return TEST_PASSED;
}
static int
RunSUTAndTest(char* sutargs, int variation_num)
{
int success, sut_running, return_code;
char hash[33];
SDL_Event event;
return_code = TEST_PASSED;
if(!sutargs)
{
SDLTest_LogError("sutargs argument cannot be NULL");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_generic;
}
SDLVisualTest_HashString(sutargs, hash);
SDLTest_Log("Hash: %s", hash);
success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
if(!success)
{
SDLTest_Log("Could not launch SUT.");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_generic;
}
SDLTest_Log("SUT launch successful.");
SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
sut_running = 1;
/* launch the timers */
SDLTest_Log("Performing actions..");
current = state.action_queue.front;
action_timer = 0;
kill_timer = 0;
if(current)
{
action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
if(!action_timer)
{
SDLTest_LogError("SDL_AddTimer() failed");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_timer;
}
}
kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
if(!kill_timer)
{
SDLTest_LogError("SDL_AddTimer() failed");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_timer;
}
/* the timer stops running if the actions queue is empty, and the
SUT stops running if it crashes or if we encounter a KILL/QUIT action */
while(sut_running)
{
/* process the actions by using an event queue */
while(SDL_PollEvent(&event))
{
if(event.type == SDL_USEREVENT)
{
if(event.user.code == ACTION_TIMER_EVENT)
{
SDLVisualTest_Action* action;
action = (SDLVisualTest_Action*)event.user.data1;
switch(ProcessAction(action, &sut_running, sutargs))
{
case TEST_PASSED:
break;
case TEST_FAILED:
return_code = TEST_FAILED;
goto runsutandtest_cleanup_timer;
break;
default:
SDLTest_LogError("ProcessAction() failed");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_timer;
}
}
else if(event.user.code == KILL_TIMER_EVENT)
{
SDLTest_LogError("Maximum timeout reached. Force killing..");
return_code = TEST_FAILED;
goto runsutandtest_cleanup_timer;
}
}
else if(event.type == SDL_QUIT)
{
SDLTest_LogError("Received QUIT event. Testharness is quitting..");
return_code = TEST_ERROR;
goto runsutandtest_cleanup_timer;
}
}
SDL_Delay(1000/ACTION_LOOP_FPS);
}
SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
if(sut_exitstatus.exit_status == 0)
{
return_code = TEST_PASSED;
goto runsutandtest_cleanup_timer;
}
else
{
return_code = TEST_FAILED;
goto runsutandtest_cleanup_timer;
}
return_code = TEST_ERROR;
goto runsutandtest_cleanup_generic;
runsutandtest_cleanup_timer:
if(action_timer && !SDL_RemoveTimer(action_timer))
{
SDLTest_Log("SDL_RemoveTimer() failed");
return_code = TEST_ERROR;
}
if(kill_timer && !SDL_RemoveTimer(kill_timer))
{
SDLTest_Log("SDL_RemoveTimer() failed");
return_code = TEST_ERROR;
}
/* runsutandtest_cleanup_process: */
if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
{
SDLTest_Log("SDL_KillProcess() failed");
return_code = TEST_ERROR;
}
runsutandtest_cleanup_generic:
return return_code;
}
/** Entry point for testharness */
int
main(int argc, char* argv[])
{
int i, passed, return_code, failed;
/* freeing resources, linux style! */
return_code = 0;
if(argc < 2)
{
SDLTest_Log(usage(), argv[0]);
goto cleanup_generic;
}
#if defined(__LINUX__) || defined(__CYGWIN__)
signal(SIGINT, CtrlCHandlerCallback);
#endif
/* parse arguments */
if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
{
SDLTest_Log(usage(), argv[0]);
return_code = 1;
goto cleanup_generic;
}
SDLTest_Log("Parsed harness arguments successfully.");
/* initialize SDL */
if(SDL_Init(SDL_INIT_TIMER) == -1)
{
SDLTest_LogError("SDL_Init() failed.");
SDLVisualTest_FreeHarnessState(&state);
return_code = 1;
goto cleanup_harness_state;
}
/* create an output directory if none exists */
#if defined(__LINUX__) || defined(__CYGWIN__)
mkdir(state.output_dir, 0777);
#elif defined(__WIN32__)
_mkdir(state.output_dir);
#else
#error "Unsupported platform"
#endif
/* test with sutargs */
if(SDL_strlen(state.sutargs))
{
SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
if(!state.no_launch)
{
switch(RunSUTAndTest(state.sutargs, 0))
{
case TEST_PASSED:
SDLTest_Log("Status: PASSED");
break;
case TEST_FAILED:
SDLTest_Log("Status: FAILED");
break;
case TEST_ERROR:
SDLTest_LogError("Some error occurred while testing.");
return_code = 1;
goto cleanup_sdl;
break;
}
}
}
if(state.sut_config.num_options > 0)
{
char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
"RANDOM" : "EXHAUSTIVE";
if(state.num_variations > 0)
SDLTest_Log("Testing SUT with variator: %s for %d variations",
variator_name, state.num_variations);
else
SDLTest_Log("Testing SUT with variator: %s and ALL variations",
variator_name);
/* initialize the variator */
if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
state.variator_type, 0))
{
SDLTest_LogError("Could not initialize variator");
return_code = 1;
goto cleanup_sdl;
}
/* iterate through all the variations */
passed = 0;
failed = 0;
for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
{
char* args = SDLVisualTest_GetNextVariation(&variator);
if(!args)
break;
SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
if(!state.no_launch)
{
switch(RunSUTAndTest(args, i + 1))
{
case TEST_PASSED:
SDLTest_Log("Status: PASSED");
passed++;
break;
case TEST_FAILED:
SDLTest_Log("Status: FAILED");
failed++;
break;
case TEST_ERROR:
SDLTest_LogError("Some error occurred while testing.");
goto cleanup_variator;
break;
}
}
}
if(!state.no_launch)
{
/* report stats */
SDLTest_Log("Testing complete.");
SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
}
goto cleanup_variator;
}
goto cleanup_sdl;
cleanup_variator:
SDLVisualTest_FreeVariator(&variator);
cleanup_sdl:
SDL_Quit();
cleanup_harness_state:
SDLVisualTest_FreeHarnessState(&state);
cleanup_generic:
return return_code;
}