00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026 #include <stdio.h>
00027 #include <stdlib.h>
00028 #include <getopt.h>
00029 #include <errno.h>
00030 #include <string.h>
00031 #include <fcntl.h>
00032 #include <unistd.h>
00033 #include <sys/types.h>
00034 #include <sys/stat.h>
00035 #include <libxml/parser.h>
00036 #include <libxml/tree.h>
00037 #include <signal.h>
00038 #include <ctype.h>
00039
00040 #include "testrunnerlite.h"
00041 #include "testdefinitionparser.h"
00042 #include "testresultlogger.h"
00043 #include "testfilters.h"
00044 #include "executor.h"
00045 #include "remote_executor.h"
00046 #include "manual_executor.h"
00047 #include "utils.h"
00048 #include "log.h"
00049
00050
00051
00052
00053
00054
00055
00056 extern char* optarg;
00057
00058
00059
00060
00061
00062
00063
00064 struct timeval created;
00065 testrunner_lite_options opts;
00066 char *global_failure = NULL;
00067 int bail_out = 0;
00068
00069
00070
00071
00072
00073
00074
00075
00076
00077
00078 LOCAL td_td *current_td = NULL;
00079 LOCAL td_suite *current_suite = NULL;
00080 LOCAL td_set *current_set = NULL;
00081 LOCAL xmlChar *cur_case_name = "";
00082 LOCAL int cur_step_num;
00083
00084 LOCAL int passcount = 0;
00085 LOCAL int casecount = 0;
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097 LOCAL void process_td(td_td *);
00098
00099 LOCAL void end_td();
00100
00101 LOCAL void process_hwiddetect();
00102
00103 LOCAL void process_suite(td_suite *);
00104
00105 LOCAL void process_set(td_set *);
00106
00107 LOCAL int process_case (const void *, const void *);
00108
00109 LOCAL int case_result_na (const void *, const void *);
00110
00111 LOCAL int process_get (const void *, const void *);
00112
00113 LOCAL int step_execute (const void *, const void *);
00114
00115 LOCAL int prepost_steps_execute (const void *, const void *);
00116
00117 LOCAL int step_result_na (const void *, const void *);
00118
00119 LOCAL int step_post_process (const void *, const void *);
00120
00121
00122
00123
00124
00125
00126
00132 LOCAL int step_execute (const void *data, const void *user)
00133 {
00134 int res = CASE_PASS;
00135 td_step *step = (td_step *)data;
00136 td_case *c = (td_case *)user;
00137 exec_data edata;
00138
00139 cur_step_num++;
00140
00141 memset (&edata, 0x0, sizeof (exec_data));
00142 if (bail_out) {
00143 res = CASE_FAIL;
00144 step->has_result = 1;
00145 step->return_code = bail_out;
00146 if (global_failure) {
00147 step->failure_info = xmlCharStrdup (global_failure);
00148 c->failure_info = xmlCharStrdup (global_failure);
00149 }
00150 goto out;
00151 }
00152
00153 if (step->manual) {
00154 if (c->dummy) {
00155 LOG_MSG (LOG_WARNING,
00156 "manual pre/post steps not supported");
00157 goto out;
00158 }
00159 if (!c->gen.manual)
00160 LOG_MSG (LOG_WARNING, "Executing manual step from "
00161 "automatic case %s "
00162 "(generally not a good idea)",
00163 c->gen.name);
00164 res = execute_manual (step);
00165 goto out;
00166 }
00167
00168 init_exec_data(&edata);
00169
00170 if (c->dummy) {
00171
00172 edata.redirect_output = DONT_REDIRECT_OUTPUT;
00173 } else {
00174 edata.redirect_output = REDIRECT_OUTPUT;
00175 }
00176 edata.soft_timeout = c->gen.timeout;
00177 edata.hard_timeout = COMMON_HARD_TIMEOUT;
00178
00179 if (step->step) {
00180 execute((char*)step->step, &edata);
00181
00182 if (step->stdout_) free (step->stdout_);
00183 if (step->stderr_) free (step->stderr_);
00184 if (step->failure_info) free (step->failure_info);
00185
00186 if (edata.stdout_data.buffer) {
00187 step->stdout_ = edata.stdout_data.buffer;
00188 }
00189 if (edata.stderr_data.buffer) {
00190 step->stderr_ = edata.stderr_data.buffer;
00191 }
00192 if (edata.failure_info.buffer) {
00193 step->failure_info = edata.failure_info.buffer;
00194 c->failure_info = xmlCharStrdup ((char *)
00195 step->failure_info);
00196
00197 LOG_MSG (LOG_INFO, "FAILURE INFO: %s",
00198 step->failure_info);
00199 }
00200
00201 step->pgid = edata.pgid;
00202 step->pid = edata.pid;
00203 step->has_result = 1;
00204 step->return_code = edata.result;
00205 step->start = edata.start_time;
00206 step->end = edata.end_time;
00207
00208
00209
00210
00211 if (c->dummy) {
00212 if (step->has_expected_result &&
00213 (step->return_code != step->expected_result)) {
00214 LOG_MSG (LOG_INFO,
00215 "STEP: %s return %d expected %d\n",
00216 step->step, step->return_code,
00217 step->expected_result);
00218 res = CASE_FAIL;
00219 }
00220 } else if (step->return_code != step->expected_result) {
00221 LOG_MSG (LOG_INFO, "STEP: %s return %d expected %d\n",
00222 step->step, step->return_code,
00223 step->expected_result);
00224 res = CASE_FAIL;
00225 }
00226 }
00227 out:
00228 if (res != CASE_PASS)
00229 c->case_res = res;
00230
00231
00232 return (res == CASE_PASS);
00233 }
00234
00235
00241 LOCAL int prepost_steps_execute (const void *data, const void *user)
00242 {
00243 td_steps *steps = (td_steps *)data;
00244 td_case *dummy = (td_case *)user;
00245
00246 if (steps->timeout == 0) {
00247 dummy->gen.timeout = 180;
00248 } else {
00249 dummy->gen.timeout = steps->timeout;
00250 }
00251
00252 if (xmlListSize(steps->steps) > 0) {
00253 xmlListWalk (steps->steps, step_execute, dummy);
00254 }
00255
00256 return 1;
00257 }
00258
00259
00265 LOCAL int step_result_na (const void *data, const void *user)
00266 {
00267 td_step *step = (td_step *)data;
00268 char *failure_info = (char *)user;
00269
00270 step->has_result = 1;
00271 step->return_code = step->expected_result + 255;
00272 step->failure_info = xmlCharStrdup (failure_info);
00273
00274 return 1;
00275 }
00276
00277
00284 LOCAL int step_post_process (const void *data, const void *user)
00285 {
00286 td_step *step = (td_step *)data;
00287 td_case *c = (td_case *)user;
00288
00289
00290 if (step->manual)
00291 goto out;
00292
00293 if (c->filtered)
00294 goto out;
00295
00296
00297 if (!step->start)
00298 goto out;
00299
00300
00301 if (!step->pgid)
00302 goto out;
00303
00304 if (opts.target_address) {
00305 ssh_kill (opts.target_address, step->pid);
00306 }
00307 kill_pgroup(step->pgid, SIGKILL);
00308
00309 out:
00310 return 1;
00311 }
00312
00313
00319 LOCAL int process_case (const void *data, const void *user)
00320 {
00321
00322 td_case *c = (td_case *)data;
00323
00324 if (c->gen.manual && !opts.run_manual) {
00325 LOG_MSG(LOG_DEBUG, "Skipping manual case %s",
00326 c->gen.name);
00327 c->filtered = 1;
00328 return 1;
00329 }
00330 if (!c->gen.manual && !opts.run_automatic) {
00331 LOG_MSG(LOG_DEBUG, "Skipping automatic case %s",
00332 c->gen.name);
00333 c->filtered = 1;
00334 return 1;
00335 }
00336 if (filter_case (c)) {
00337 LOG_MSG (LOG_INFO, "Test case %s is filtered", c->gen.name);
00338 return 1;
00339 }
00340
00341 cur_case_name = c->gen.name;
00342 LOG_MSG (LOG_INFO, "Starting test case %s", c->gen.name);
00343 casecount++;
00344
00345 c->case_res = CASE_PASS;
00346 if (c->gen.timeout == 0)
00347 c->gen.timeout = COMMON_SOFT_TIMEOUT;
00348
00349 if (c->gen.manual && opts.run_manual)
00350 pre_manual (c);
00351 cur_step_num = 0;
00352 if (xmlListSize (c->steps) == 0) {
00353 LOG_MSG (LOG_WARNING, "Case with no steps (%s).",
00354 c->gen.name);
00355 }
00356
00357 xmlListWalk (c->steps, step_execute, data);
00358 xmlListWalk (c->steps, step_post_process, data);
00359
00360 if (c->gen.manual && opts.run_manual)
00361 post_manual (c);
00362
00363 LOG_MSG (LOG_INFO, "Finished test case Result: %s",
00364 case_result_str(c->case_res));
00365 passcount += (c->case_res == CASE_PASS);
00366
00367 return 1;
00368 }
00369
00375 LOCAL int case_result_na (const void *data, const void *user)
00376 {
00377
00378 td_case *c = (td_case *)data;
00379 char *failure_info = (char *)user;
00380
00381 LOG_MSG (LOG_DEBUG, "Setting FAIL result for case %s", c->gen.name);
00382
00383 c->case_res = CASE_FAIL;
00384 c->failure_info = xmlCharStrdup (failure_info);
00385
00386 xmlListWalk (c->steps, step_result_na, user);
00387
00388 return 1;
00389 }
00390
00391
00397 LOCAL int process_get (const void *data, const void *user)
00398 {
00399
00400 td_file *file = (td_file *)data;
00401 xmlChar *command;
00402 char *fname;
00403 exec_data edata;
00404 char *remote = opts.target_address;
00405
00406 memset (&edata, 0x0, sizeof (exec_data));
00407 init_exec_data(&edata);
00408 edata.soft_timeout = COMMON_SOFT_TIMEOUT;
00409 edata.hard_timeout = COMMON_HARD_TIMEOUT;
00410
00411 fname = malloc (strlen((char *)file->filename) + 1);
00412 trim_string ((char *)file->filename, fname);
00413
00414
00415
00416
00417 if (remote) {
00418 opts.target_address = NULL;
00419 command = (xmlChar *)malloc (strlen ("scp ") +
00420 strlen (fname) +
00421 strlen (opts.output_folder) +
00422 strlen (remote) + 10);
00423 sprintf ((char *)command, "scp %s:\'%s\' %s", remote, fname,
00424 opts.output_folder);
00425
00426 } else {
00427 command = (xmlChar *)malloc (strlen ("cp ") +
00428 strlen (fname) +
00429 strlen (opts.output_folder) + 2);
00430 sprintf ((char *)command, "cp %s %s", fname,
00431 opts.output_folder);
00432 }
00433 LOG_MSG (LOG_DEBUG, "%s: Executing command: %s", PROGNAME,
00434 (char*)command);
00435
00436
00437
00438 execute((char*)command, &edata);
00439
00440 if (edata.result) {
00441 LOG_MSG (LOG_ERR, "%s: %s failed: %s\n", PROGNAME, command,
00442 (char *)(edata.stderr_data.buffer ?
00443 edata.stderr_data.buffer :
00444 BAD_CAST "no info available"));
00445 }
00446 opts.target_address = remote;
00447 if (edata.stdout_data.buffer) free (edata.stdout_data.buffer);
00448 if (edata.stderr_data.buffer) free (edata.stderr_data.buffer);
00449 if (edata.failure_info.buffer) free (edata.failure_info.buffer);
00450
00451 if (!file->delete_after)
00452 goto out;
00453
00454 memset (&edata, 0x0, sizeof (exec_data));
00455 init_exec_data(&edata);
00456 edata.soft_timeout = COMMON_SOFT_TIMEOUT;
00457 edata.hard_timeout = COMMON_HARD_TIMEOUT;
00458 sprintf ((char *)command, "rm -f %s", fname);
00459 LOG_MSG (LOG_DEBUG, "%s: Executing command: %s", PROGNAME,
00460 (char*)command);
00461 execute((char*)command, &edata);
00462 if (edata.result) {
00463 LOG_MSG (LOG_ERR, "%s: %s failed: %s\n", PROGNAME, command,
00464 (char *)(edata.stderr_data.buffer ?
00465 edata.stderr_data.buffer :
00466 BAD_CAST "no info available"));
00467 }
00468 if (edata.stdout_data.buffer) free (edata.stdout_data.buffer);
00469 if (edata.stderr_data.buffer) free (edata.stderr_data.buffer);
00470 if (edata.failure_info.buffer) free (edata.failure_info.buffer);
00471
00472 out:
00473 free (command);
00474 free (fname);
00475 return 1;
00476 }
00477
00481 LOCAL void process_td (td_td *td)
00482 {
00483 current_td = td;
00484 }
00485
00488 LOCAL void end_td ()
00489 {
00490 td_td_delete (current_td);
00491 current_td = NULL;
00492 }
00493
00496 LOCAL void process_hwiddetect ()
00497 {
00498 exec_data edata;
00499 char* trimmed = NULL;
00500 size_t length = 0;
00501
00502 if (current_td && current_td->hw_detector) {
00503 init_exec_data(&edata);
00504 edata.redirect_output = REDIRECT_OUTPUT;
00505 edata.soft_timeout = COMMON_SOFT_TIMEOUT;
00506 edata.hard_timeout = COMMON_HARD_TIMEOUT;
00507
00508 execute((char*)current_td->hw_detector, &edata);
00509
00510 if (edata.result != EXIT_SUCCESS) {
00511 LOG_MSG (LOG_ERR, "Running HW ID detector "
00512 "failed with return value %d",
00513 edata.result);
00514 } else if (edata.stdout_data.buffer) {
00515
00516 length = strlen((char*)edata.stdout_data.buffer);
00517 trimmed = (char*)malloc(length + 1);
00518 trim_string((char*)edata.stdout_data.buffer, trimmed);
00519
00520 current_td->detected_hw = xmlCharStrdup(trimmed);
00521 LOG_MSG (LOG_INFO, "Detected HW ID '%s'",
00522 current_td->detected_hw);
00523 }
00524
00525 clean_exec_data(&edata);
00526 }
00527
00528 free(trimmed);
00529 }
00530
00534 LOCAL void process_suite (td_suite *s)
00535 {
00536 LOG_MSG (LOG_INFO, "Test suite: %s", s->gen.name);
00537
00538 write_pre_suite_tag (s);
00539 current_suite = s;
00540
00541 }
00542
00545 LOCAL void end_suite ()
00546 {
00547 write_post_suite_tag ();
00548 td_suite_delete (current_suite);
00549 current_suite = NULL;
00550 }
00551
00555 LOCAL void process_set (td_set *s)
00556 {
00557 td_case dummy;
00558
00559
00560
00561
00562 if (filter_set (s)) {
00563 LOG_MSG (LOG_INFO, "Test set %s is filtered", s->gen.name);
00564 goto skip_all;
00565 }
00566
00567
00568
00569
00570 if (s->gen.hwid && current_td->detected_hw &&
00571 xmlStrcmp(s->gen.hwid, current_td->detected_hw) != 0) {
00572 LOG_MSG (LOG_INFO, "Test set %s is filtered based on HW ID",
00573 s->gen.name);
00574 goto skip_all;
00575 }
00576
00577
00578
00579
00580 s->environment = xmlCharStrdup (opts.environment);
00581 if (!xmlListSearch (s->environments, opts.environment)) {
00582 LOG_MSG (LOG_INFO, "Test set %s not run on "
00583 "environment: %s",
00584 s->gen.name, opts.environment);
00585 goto skip_all;
00586 }
00587 current_set = s;
00588 LOG_MSG (LOG_INFO, "Test set: %s", s->gen.name);
00589 write_pre_set_tag (s);
00590
00591 if (xmlListSize (s->pre_steps) > 0) {
00592 cur_case_name = (xmlChar *)"pre_steps";
00593 cur_step_num = 0;
00594 memset (&dummy, 0x0, sizeof (td_case));
00595 dummy.case_res = CASE_PASS;
00596 dummy.dummy = 1;
00597 LOG_MSG (LOG_INFO, "Executing pre steps");
00598 xmlListWalk (s->pre_steps, prepost_steps_execute, &dummy);
00599 if (dummy.case_res != CASE_PASS) {
00600 LOG_MSG (LOG_ERR, "Pre steps failed. "
00601 "Test set %s aborted.", s->gen.name);
00602 xmlListWalk (s->cases, case_result_na,
00603 global_failure ? global_failure :
00604 "pre_steps failed");
00605 goto short_circuit;
00606 }
00607 }
00608
00609 xmlListWalk (s->cases, process_case, s);
00610 if (xmlListSize (s->post_steps) > 0) {
00611 LOG_MSG (LOG_INFO, "Executing post steps");
00612 cur_case_name = (xmlChar *)"post_steps";
00613 cur_step_num = 0;
00614 memset (&dummy, 0x0, sizeof (td_case));
00615 dummy.case_res = CASE_PASS;
00616 dummy.dummy = 1;
00617 xmlListWalk (s->post_steps, prepost_steps_execute, &dummy);
00618 if (dummy.case_res == CASE_FAIL)
00619 LOG_MSG (LOG_ERR,
00620 "Post steps failed for %s.", s->gen.name);
00621 }
00622 xmlListWalk (s->gets, process_get, s);
00623
00624 short_circuit:
00625 write_post_set_tag (s);
00626 if (xmlListSize (s->pre_steps) > 0)
00627 xmlListWalk (s->pre_steps, step_post_process, &dummy);
00628 if (xmlListSize (s->post_steps) > 0)
00629 xmlListWalk (s->post_steps, step_post_process, &dummy);
00630 xml_end_element();
00631 skip_all:
00632 td_set_delete (s);
00633 return;
00634 }
00635
00636
00637
00641 void td_process () {
00642 int retval;
00643 td_parser_callbacks cbs;
00644
00645 memset (&cbs, 0x0, sizeof(td_parser_callbacks));
00646
00647
00648
00649 cbs.test_td = process_td;
00650 cbs.test_td_end = end_td;
00651 cbs.test_hwiddetect = process_hwiddetect;
00652 cbs.test_suite = process_suite;
00653 cbs.test_suite_end = end_suite;
00654 cbs.test_set = process_set;
00655
00656 retval = td_register_callbacks (&cbs);
00657
00658
00659
00660
00661 LOG_MSG (LOG_INFO, "Starting to run tests...");
00662
00663 while (td_next_node() == 0);
00664
00665 LOG_MSG (LOG_INFO, "Finished running tests.");
00666 LOG_MSG (LOG_INFO, "Executed %d cases. Passed %d Failed %d",
00667 casecount, passcount, casecount - passcount);
00668 return;
00669 }
00670
00674 const char *current_set_name ()
00675 {
00676 if (current_set)
00677 return (char *)current_set->gen.name;
00678 return "";
00679 }
00680
00684 const char *current_case_name ()
00685 {
00686 return (char *)cur_case_name;
00687 }
00688
00692 int current_step_num ()
00693 {
00694 return cur_step_num;
00695 }
00696
00697
00698
00699
00700
00701