Grcov report - cairo-test-runner.c

1

/*

2

 * Copyright © 2008 Chris Wilson

3

4

 * Permission to use, copy, modify, distribute, and sell this software

5

 * and its documentation for any purpose is hereby granted without

6

 * fee, provided that the above copyright notice appear in all copies

7

 * and that both that copyright notice and this permission notice

8

 * appear in supporting documentation, and that the name of

9

 * Chris Wilson not be used in advertising or publicity pertaining to

10

 * distribution of the software without specific, written prior

11

 * permission. Chris Wilson makes no representations about the

12

 * suitability of this software for any purpose.  It is provided "as

13

 * is" without express or implied warranty.

14

15

 * CHRIS WILSON DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS

16

 * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND

17

 * FITNESS, IN NO EVENT SHALL CHRIS WILSON BE LIABLE FOR ANY SPECIAL,

18

 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER

19

 * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION

20

 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR

21

 * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

22

23

 * Author: Chris Wilson <chris@chris-wilson.co.uk>

24

*/

25

26

#include "cairo-test-private.h"

27

#include "cairo-boilerplate-getopt.h"

28

29

#include <pixman.h> /* for version information */

30

31

#define SHOULD_FORK HAVE_FORK && HAVE_WAITPID

32

#if HAVE_UNISTD_H

33

#include <unistd.h>

34

#endif

35

#if SHOULD_FORK

36

#if HAVE_SIGNAL_H

37

#include <signal.h>

38

#endif

39

#include <sys/types.h>

40

#include <sys/wait.h>

41

#endif

42

#if HAVE_LIBGEN_H

43

#include <libgen.h>

44

#endif

45

46

#if HAVE_VALGRIND

47

#include <valgrind.h>

48

#else

49

#define RUNNING_ON_VALGRIND 0

50

#endif

51

52

#ifdef _MSC_VER

53

#include <crtdbg.h>

54

#endif

55

56

typedef struct _cairo_test_list {

57

    const cairo_test_t *test;

58

    struct _cairo_test_list *next;

59

} cairo_test_list_t;

60

61

typedef struct _cairo_test_runner {

62

    cairo_test_context_t base;

63

64

    unsigned int num_device_offsets;

65

    unsigned int num_device_scales;

66

67

    cairo_bool_t passed;

68

    int num_passed;

69

    int num_skipped;

70

    int num_failed;

71

    int num_xfailed;

72

    int num_error;

73

    int num_crashed;

74

75

    unsigned int num_ignored_via_env;

76

77

    cairo_test_list_t *crashes_preamble;

78

    cairo_test_list_t *errors_preamble;

79

    cairo_test_list_t *fails_preamble;

80

81

    cairo_test_list_t **crashes_per_target;

82

    cairo_test_list_t **errors_per_target;

83

    cairo_test_list_t **fails_per_target;

84

85

    int *num_failed_per_target;

86

    int *num_error_per_target;

87

    int *num_crashed_per_target;

88

89

    cairo_bool_t foreground;

90

    cairo_bool_t exit_on_failure;

91

    cairo_bool_t list_only;

92

    cairo_bool_t full_test;

93

    cairo_bool_t keyword_match;

94

    cairo_bool_t slow;

95

    cairo_bool_t force_pass;

96

} cairo_test_runner_t;

97

98

typedef enum {

99

GE,

100

GT

101

} cairo_test_compare_op_t;

102

103

static cairo_test_list_t *tests;

104

105

static void CAIRO_BOILERPLATE_PRINTF_FORMAT(2,3)

106

643

_log (cairo_test_context_t *ctx,

107

      const char *fmt,

108

      ...)

109

110

    va_list ap;

111

112

643

    va_start (ap, fmt);

113

643

    vprintf (fmt, ap);

114

643

    va_end (ap);

115

116

643

    va_start (ap, fmt);

117

643

    cairo_test_logv (ctx, fmt, ap);

118

643

    va_end (ap);

119

643

120

121

static cairo_test_list_t *

122

632

_list_prepend (cairo_test_list_t *head, const cairo_test_t *test)

123

124

    cairo_test_list_t *list;

125

126

632

    list = xmalloc (sizeof (cairo_test_list_t));

127

632

    list->test = test;

128

632

    list->next = head;

129

632

    head = list;

130

131

632

    return head;

132

133

134

static cairo_test_list_t *

135

10

_list_reverse (cairo_test_list_t *head)

136

137

    cairo_test_list_t *list, *next;

138

139

642

    for (list = head, head = NULL; list != NULL; list = next) {

140

632

	next = list->next;

141

632

	list->next = head;

142

632

	head = list;

143

144

145

10

    return head;

146

147

148

static void

149

13

_list_free (cairo_test_list_t *list)

150

151

645

    while (list != NULL) {

152

632

	cairo_test_list_t *next = list->next;

153

632

	free (list);

154

632

	list = next;

155

156

13

157

158

static cairo_bool_t

159

1

is_running_under_debugger (void)

160

161

#if HAVE_UNISTD_H && HAVE_LIBGEN_H && __linux__

162

1

    char buf[1024] = { 0 };

163

1

    char buf2[1024] = { 0 };

164

165

1

    sprintf (buf, "/proc/%d/exe", getppid ());

166

1

    if (readlink (buf, buf2, sizeof (buf2)) != -1 &&

167

1

	buf2[1023] == 0 &&

168

1

	strncmp (basename (buf2), "gdb", 3) == 0)

169

170

	return TRUE;

171

172

#endif

173

174

    if (RUNNING_ON_VALGRIND)

175

	return TRUE;

176

177

1

    return FALSE;

178

179

180

#if SHOULD_FORK

181

static cairo_test_status_t

182

1766

_cairo_test_wait (pid_t pid)

183

184

    int exitcode;

185

186

1766

    if (waitpid (pid, &exitcode, 0) != pid)

187

	return CAIRO_TEST_CRASHED;

188

189

1766

    if (WIFSIGNALED (exitcode)) {

190

	switch (WTERMSIG (exitcode)) {

191

	case SIGINT:

192

#if HAVE_RAISE

193

	    raise (SIGINT);

194

#endif

195

	    return CAIRO_TEST_UNTESTED;

196

	default:

197

	    return CAIRO_TEST_CRASHED;

198

199

200

201

1766

    return WEXITSTATUS (exitcode);

202

203

#endif

204

205

static cairo_test_status_t

206

44

_cairo_test_runner_preamble (cairo_test_runner_t *runner,

207

			     cairo_test_context_t *ctx)

208

209

#if SHOULD_FORK

210

44

    if (! runner->foreground) {

211

	pid_t pid;

212

213

	/* fork() duplicates output buffers, so clear them */

214

44

	fflush (NULL);

215

216

44

	switch ((pid = fork ())) {

217

	case -1: /* error */

218

	    return CAIRO_TEST_UNTESTED;

219

220

44

	case 0: /* child */

221

44

	    exit (ctx->test->preamble (ctx));

222

223

44

	default:

224

44

	    return _cairo_test_wait (pid);

225

226

227

#endif

228

    return ctx->test->preamble (ctx);

229

230

231

static cairo_test_status_t

232

1722

_cairo_test_runner_draw (cairo_test_runner_t *runner,

233

			 cairo_test_context_t *ctx,

234

			 const cairo_boilerplate_target_t *target,

235

			 cairo_bool_t similar,

236

			 int device_offset, int device_scale)

237

238

#if SHOULD_FORK

239

1722

    if (! runner->foreground) {

240

	pid_t pid;

241

242

	/* fork() duplicates output buffers, so clear them */

243

1722

	fflush (NULL);

244

245

1722

	switch ((pid = fork ())) {

246

	case -1: /* error */

247

	    return CAIRO_TEST_UNTESTED;

248

249

1722

	case 0: /* child */

250

1722

	    exit (_cairo_test_context_run_for_target (ctx, target,

251

						      similar, device_offset, device_scale));

252

253

1722

	default:

254

1722

	    return _cairo_test_wait (pid);

255

256

257

#endif

258

    return _cairo_test_context_run_for_target (ctx, target,

259

					       similar, device_offset, device_scale);

260

261

262

static void

263

1

append_argv (int *argc, char ***argv, const char *str)

264

265

    int old_argc;

266

    char **old_argv;

267

    cairo_bool_t doit;

268

    const char *s, *t;

269

    int olen;

270

    int len;

271

    int i;

272

1

    int args_to_add = 0;

273

274

1

    if (str == NULL)

275

1

	return;

276

277

    old_argc = *argc;

278

    old_argv = *argv;

279

280

    doit = FALSE;

281

    do {

282

	if (doit)

283

	    *argv = xmalloc (olen);

284

285

	olen = sizeof (char *) * (args_to_add + *argc);

286

	for (i = 0; i < old_argc; i++) {

287

	    len = strlen (old_argv[i]) + 1;

288

	    if (doit) {

289

		(*argv)[i] = (char *) *argv + olen;

290

		memcpy ((*argv)[i], old_argv[i], len);

291

292

	    olen += len;

293

294

295

	s = str;

296

	while ((t = strpbrk (s, " \t,:;")) != NULL) {

297

	    if (t - s) {

298

		len = t - s;

299

		if (doit) {

300

		    (*argv)[i] = (char *) *argv + olen;

301

		    memcpy ((*argv)[i], s, len);

302

		    (*argv)[i][len] = '\0';

303

		} else {

304

		    olen += sizeof (char *);

305

306

		args_to_add++;

307

		olen += len + 1;

308

		i++;

309

310

	    s = t + 1;

311

312

	if (*s != '\0') {

313

	    len = strlen (s) + 1;

314

	    if (doit) {

315

		(*argv)[i] = (char *) *argv + olen;

316

		memcpy ((*argv)[i], s, len);

317

	    } else {

318

		olen += sizeof (char *);

319

320

	    args_to_add++;

321

	    olen += len;

322

	    i++;

323

324

    } while (doit++ == FALSE);

325

    *argc = i;

326

327

328

static void

329

usage (const char *argv0)

330

331

    fprintf (stderr,

332

	     "Usage: %s [-afkxsl] [test-names|keywords ...]\n"

333

	     "\n"

334

	     "Run the cairo conformance test suite over the given tests (all by default)\n"

335

	     "The command-line arguments are interpreted as follows:\n"

336

	     "\n"

337

	     "  -a	all; run the full set of tests. By default the test suite\n"

338

	     "          skips similar surface and device offset testing.\n"

339

	     "  -f	foreground; do not fork\n"

340

	     "  -k	match tests by keyword\n"

341

	     "  -l	list only; just list selected test case names without executing\n"

342

	     "  -s	include slow, long running tests\n"

343

	     "  -x	exit on first failure\n"

344

	     "\n"

345

	     "If test names are given they are used as matches either to a specific\n"

346

	     "test case or to a keyword, so a command such as\n"

347

	     "\"%s -k text\" can be used to run all text test cases, and\n"

348

	     "\"%s text-transform\" to run the individual case.\n",

349

	     argv0, argv0, argv0);

350

351

352

static void

353

1

_parse_cmdline (cairo_test_runner_t *runner, int *argc, char **argv[])

354

355

    int c;

356

357

    while (1) {

358

1

	c = _cairo_getopt (*argc, *argv, ":afklsx");

359

1

	if (c == -1)

360

1

	    break;

361

362

	switch (c) {

363

	case 'a':

364

	    runner->full_test = ~0;

365

	    break;

366

	case 'f':

367

	    runner->foreground = TRUE;

368

	    break;

369

	case 'k':

370

	    runner->keyword_match = TRUE;

371

	    break;

372

	case 'l':

373

	    runner->list_only = TRUE;

374

	    break;

375

	case 's':

376

	    runner->slow = TRUE;

377

	    break;

378

	case 'x':

379

	    runner->exit_on_failure = TRUE;

380

	    break;

381

	default:

382

	    fprintf (stderr, "Internal error: unhandled option: %c\n", c);

383

	    /* fall-through */

384

	case '?':

385

	    usage ((*argv)[0]);

386

	    exit (1);

387

388

389

390

1

    *argc -= optind;

391

1

    *argv += optind;

392

1

393

394

static void

395

1

_runner_init (cairo_test_runner_t *runner)

396

397

1

    cairo_test_init (&runner->base, "cairo-test-suite", ".");

398

399

1

    runner->passed = TRUE;

400

401

1

    runner->fails_preamble = NULL;

402

1

    runner->crashes_preamble = NULL;

403

1

    runner->errors_preamble = NULL;

404

405

1

    runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *),

406

					runner->base.num_targets);

407

1

    runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *),

408

					  runner->base.num_targets);

409

1

    runner->errors_per_target = xcalloc (sizeof (cairo_test_list_t *),

410

					  runner->base.num_targets);

411

1

    runner->num_failed_per_target = xcalloc (sizeof (int),

412

					     runner->base.num_targets);

413

1

    runner->num_error_per_target = xcalloc (sizeof (int),

414

					     runner->base.num_targets);

415

1

    runner->num_crashed_per_target = xcalloc (sizeof (int),

416

					      runner->base.num_targets);

417

1

418

419

static void

420

1

_runner_print_versions (cairo_test_runner_t *runner)

421

422

1

    _log (&runner->base,

423

	 "Compiled against cairo %s, running on %s.\n",

424

	 CAIRO_VERSION_STRING, cairo_version_string ());

425

1

    _log (&runner->base,

426

	 "Compiled against pixman %s, running on %s.\n",

427

	 PIXMAN_VERSION_STRING, pixman_version_string ());

428

429

1

    fflush (runner->base.log_file);

430

1

431

432

static void

433

1

_runner_print_summary (cairo_test_runner_t *runner)

434

435

1

    _log (&runner->base,

436

	  "%d Passed, %d Failed [%d crashed, %d expected], %d Skipped\n",

437

	  runner->num_passed,

438

439

1

	  runner->num_failed + runner->num_crashed + runner->num_xfailed,

440

	  runner->num_crashed,

441

	  runner->num_xfailed,

442

443

	  runner->num_skipped);

444

1

    if (runner->num_ignored_via_env) {

445

1

	_log (&runner->base,

446

	      "%d expected failure due to special request via environment variables!\n",

447

	      runner->num_ignored_via_env);

448

449

1

450

451

static void

452

1

_runner_print_details (cairo_test_runner_t *runner)

453

454

    cairo_test_list_t *list;

455

    unsigned int n;

456

457

1

    if (runner->crashes_preamble) {

458

	int count = 0;

459

460

	for (list = runner->crashes_preamble; list != NULL; list = list->next)

461

	    count++;

462

463

	_log (&runner->base, "Preamble: %d crashed! -", count);

464

465

	for (list = runner->crashes_preamble; list != NULL; list = list->next) {

466

	    char *name = cairo_test_get_name (list->test);

467

	    _log (&runner->base, " %s", name);

468

	    free (name);

469

470

	_log (&runner->base, "\n");

471

472

1

    if (runner->errors_preamble) {

473

	int count = 0;

474

475

	for (list = runner->errors_preamble; list != NULL; list = list->next)

476

	    count++;

477

478

	_log (&runner->base, "Preamble: %d error -", count);

479

480

	for (list = runner->errors_preamble; list != NULL; list = list->next) {

481

	    char *name = cairo_test_get_name (list->test);

482

	    _log (&runner->base, " %s", name);

483

	    free (name);

484

485

	_log (&runner->base, "\n");

486

487

1

    if (runner->fails_preamble) {

488

	int count = 0;

489

490

	for (list = runner->fails_preamble; list != NULL; list = list->next)

491

	    count++;

492

493

	_log (&runner->base, "Preamble: %d failed -", count);

494

495

	for (list = runner->fails_preamble; list != NULL; list = list->next) {

496

	    char *name = cairo_test_get_name (list->test);

497

	    _log (&runner->base, " %s", name);

498

	    free (name);

499

500

	_log (&runner->base, "\n");

501

502

503

4

    for (n = 0; n < runner->base.num_targets; n++) {

504

	const cairo_boilerplate_target_t *target;

505

506

3

	target = runner->base.targets_to_test[n];

507

3

	if (runner->num_crashed_per_target[n]) {

508

	    _log (&runner->base, "%s (%s): %d crashed! -",

509

		  target->name,

510

		  cairo_boilerplate_content_name (target->content),

511

		  runner->num_crashed_per_target[n]);

512

513

	    for (list = runner->crashes_per_target[n];

514

		 list != NULL;

515

		 list = list->next)

516

517

		char *name = cairo_test_get_name (list->test);

518

		_log (&runner->base, " %s", name);

519

		free (name);

520

521

	    _log (&runner->base, "\n");

522

523

3

	if (runner->num_error_per_target[n]) {

524

6

	    _log (&runner->base, "%s (%s): %d error -",

525

3

		  target->name,

526

3

		  cairo_boilerplate_content_name (target->content),

527

3

		  runner->num_error_per_target[n]);

528

529

3

	    for (list = runner->errors_per_target[n];

530

14

		 list != NULL;

531

11

		 list = list->next)

532

533

11

		char *name = cairo_test_get_name (list->test);

534

11

		_log (&runner->base, " %s", name);

535

11

		free (name);

536

537

3

	    _log (&runner->base, "\n");

538

539

540

3

	if (runner->num_failed_per_target[n]) {

541

	    _log (&runner->base, "%s (%s): %d failed -",

542

		  target->name,

543

		  cairo_boilerplate_content_name (target->content),

544

		  runner->num_failed_per_target[n]);

545

546

	    for (list = runner->fails_per_target[n];

547

		 list != NULL;

548

		 list = list->next)

549

550

		char *name = cairo_test_get_name (list->test);

551

		_log (&runner->base, " %s", name);

552

		free (name);

553

554

	    _log (&runner->base, "\n");

555

556

557

1

558

559

static void

560

1

_runner_print_results (cairo_test_runner_t *runner)

561

562

1

    _runner_print_summary (runner);

563

1

    _runner_print_details (runner);

564

565

1

    if (! runner->passed && ! runner->num_crashed) {

566

1

	_log (&runner->base,

567

"\n"

568

"Note: These failures may be due to external factors.\n"

569

"Please read test/README -- \"Getting the elusive zero failures\".\n");

570

571

1

572

573

static cairo_test_status_t

574

1

_runner_fini (cairo_test_runner_t *runner)

575

576

    unsigned int n;

577

578

1

    _list_free (runner->crashes_preamble);

579

1

    _list_free (runner->errors_preamble);

580

1

    _list_free (runner->fails_preamble);

581

582

4

    for (n = 0; n < runner->base.num_targets; n++) {

583

3

	_list_free (runner->crashes_per_target[n]);

584

3

	_list_free (runner->errors_per_target[n]);

585

3

	_list_free (runner->fails_per_target[n]);

586

587

1

    free (runner->crashes_per_target);

588

1

    free (runner->errors_per_target);

589

1

    free (runner->fails_per_target);

590

591

1

    free (runner->num_crashed_per_target);

592

1

    free (runner->num_error_per_target);

593

1

    free (runner->num_failed_per_target);

594

595

1

    cairo_test_fini (&runner->base);

596

597

1

    if (runner->force_pass)

598

	return CAIRO_TEST_SUCCESS;

599

600

1

    return runner->num_failed + runner->num_crashed ?

601

2

	CAIRO_TEST_FAILURE :

602

1

	runner->num_passed + runner->num_xfailed ?

603

	CAIRO_TEST_SUCCESS : CAIRO_TEST_UNTESTED;

604

605

606

static cairo_bool_t

607

1722

expect_fail_due_to_env_var (cairo_test_context_t *ctx,

608

                            const cairo_boilerplate_target_t *target)

609

610

1722

    const char *prefix = "CAIRO_TEST_IGNORE_";

611

1722

    const char *content = cairo_boilerplate_content_name (target->content);

612

    char *env_name;

613

    const char *env;

614

1722

    cairo_bool_t result = FALSE;

615

    char *to_replace;

616

617

    /* Construct the name of the env var */

618

1722

    env_name = malloc (strlen (prefix) + strlen (target->name) + 1 + strlen (content) + 1);

619

1722

    if (env_name == NULL) {

620

	fprintf(stderr, "Malloc failed, cannot check $%s%s_%s\n", prefix, target->name, content);

621

	cairo_test_log(ctx, "Malloc failed, cannot check $%s%s_%s\n", prefix, target->name, content);

622

	return FALSE;

623

624

1722

    strcpy (env_name, prefix);

625

1722

    strcat (env_name, target->name);

626

1722

    strcat (env_name, "_");

627

1722

    strcat (env_name, content);

628

629

    /* Deal with some invalid characters: Replace '-' and '&' with '_' */

630

1722

    while ((to_replace = strchr(env_name, '-')) != NULL) {

631

	*to_replace = '_';

632

633

1722

    while ((to_replace = strchr(env_name, '&')) != NULL) {

634

	*to_replace = '_';

635

636

637

1722

    env = getenv (env_name);

638

639

    /* Look for the test name in the env var (comma separated) */

640

1722

    if (env) {

641

10523

	for (size_t start = 0;; start += strlen (ctx->test_name)) {

642

10523

	   char *match = strstr (env + start, ctx->test_name);

643

10523

	   if (!match)

644

1345

	       break;

645

646

	   /* Make sure that "foo" does not match in "barfoo,foobaz"

647

	    * There must be commas around the test name (or string begin/end)

648

*/

649

9178

	   if (env == match || match[-1] == ',') {

650

4224

	       char *end = match + strlen (ctx->test_name);

651

4224

	       if (*end == '\0' || *end == ',') {

652

377

		   result = TRUE;

653

377

		   break;

654

655

656

657

658

1722

    if (result)

659

377

       cairo_test_log (ctx,

660

                       "Expecting '%s' to fail because it appears in $%s\n",

661

                       ctx->test_name, env_name);

662

663

1722

    free (env_name);

664

665

1722

    return result;

666

667

668

669

static cairo_bool_t

670

4

_version_compare (int a, cairo_test_compare_op_t op, int b)

671

672

4

    switch (op) {

673

    case GT: return a > b;

674

4

    case GE: return a >= b;

675

    default: return FALSE;

676

677

678

679

680

static cairo_bool_t

681

4

_get_required_version (const char *str,

682

		       cairo_test_compare_op_t *op,

683

		       int *major,

684

		       int *minor,

685

		       int *micro)

686

687

8

    while (*str == ' ')

688

4

	str++;

689

690

4

    if (strncmp (str, ">=", 2) == 0) {

691

4

	*op = GE;

692

4

	str += 2;

693

    } else if (strncmp (str, ">", 1) == 0) {

694

	*op = GT;

695

	str += 1;

696

    } else

697

	return FALSE;

698

699

8

    while (*str == ' ')

700

4

	str++;

701

702

4

    if (sscanf (str, "%d.%d.%d", major, minor, micro) != 3) {

703

	*micro = 0;

704

	if (sscanf (str, "%d.%d", major, minor) != 2)

705

	    return FALSE;

706

707

708

4

    return TRUE;

709

710

711

static cairo_bool_t

712

4

_has_required_cairo_version (const char *str)

713

714

    cairo_test_compare_op_t op;

715

    int major, minor, micro;

716

717

4

    if (! _get_required_version (str + 5 /* advance over "cairo" */,

718

				 &op, &major, &minor, &micro))

719

720

	fprintf (stderr, "unrecognised cairo version requirement '%s'\n", str);

721

	return FALSE;

722

723

724

4

    return _version_compare (cairo_version (),

725

op,

726

4

			     CAIRO_VERSION_ENCODE (major, minor, micro));

727

728

729

#define TEST_SIMILAR	0x1

730

#define TEST_OFFSET	0x2

731

#define TEST_SCALE	0x4

732

int

733

1

main (int argc, char **argv)

734

735

    cairo_test_runner_t runner;

736

    cairo_test_list_t *test_list;

737

    cairo_test_status_t *target_status;

738

    unsigned int n, m, k;

739

    char targets[4096];

740

    int len;

741

    char *cairo_tests_env;

742

743

#ifdef _MSC_VER

744

    /* We don't want an assert dialog, we want stderr */

745

    _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);

746

    _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);

747

#endif

748

749

1

    _cairo_test_runner_register_tests ();

750

1

    tests = _list_reverse (tests);

751

752

1

    memset (&runner, 0, sizeof (runner));

753

1

    runner.num_device_offsets = 1;

754

1

    runner.num_device_scales = 1;

755

756

1

    if (is_running_under_debugger ())

757

	runner.foreground = TRUE;

758

759

1

    if (getenv ("CAIRO_TEST_MODE")) {

760

	const char *env = getenv ("CAIRO_TEST_MODE");

761

762

	if (strstr (env, "full")) {

763

	    runner.full_test = ~0;

764

765

	if (strstr (env, "similar")) {

766

	    runner.full_test |= TEST_SIMILAR;

767

768

	if (strstr (env, "offset")) {

769

	    runner.full_test |= TEST_OFFSET;

770

771

	if (strstr (env, "scale")) {

772

	    runner.full_test |= TEST_SCALE;

773

774

	if (strstr (env, "foreground")) {

775

	    runner.foreground = TRUE;

776

777

	if (strstr (env, "exit-on-failure")) {

778

	    runner.exit_on_failure = TRUE;

779

780

781

782

1

    if (getenv ("CAIRO_TEST_FORCE_PASS")) {

783

	const char *env = getenv ("CAIRO_TEST_FORCE_PASS");

784

785

	runner.force_pass = atoi (env);

786

787

788

1

    _parse_cmdline (&runner, &argc, &argv);

789

790

1

    cairo_tests_env = getenv("CAIRO_TESTS");

791

1

    append_argv (&argc, &argv, cairo_tests_env);

792

793

1

    if (runner.full_test & TEST_OFFSET) {

794

	runner.num_device_offsets = 2;

795

796

1

    if (runner.full_test & TEST_SCALE) {

797

	runner.num_device_scales = 2;

798

799

800

1

    target_status = NULL; /* silence the compiler */

801

1

    if (! runner.list_only) {

802

1

	_runner_init (&runner);

803

1

	_runner_print_versions (&runner);

804

1

	target_status = xmalloc (sizeof (cairo_test_status_t) *

805

1

				 runner.base.num_targets);

806

807

808

622

    for (test_list = tests; test_list != NULL; test_list = test_list->next) {

809

621

	const cairo_test_t *test = test_list->test;

810

	cairo_test_context_t ctx;

811

	cairo_test_status_t status;

812

621

	cairo_bool_t failed = FALSE, xfailed = FALSE, error = FALSE, crashed = FALSE, skipped = TRUE;

813

621

	cairo_bool_t in_preamble = FALSE;

814

621

	char *name = cairo_test_get_name (test);

815

	int i;

816

817

	/* check for restricted runs */

818

621

	if (argc) {

819

	    cairo_bool_t found = FALSE;

820

	    const char *keywords = test->keywords;

821

822

	    for (i = 0; i < argc; i++) {

823

		const char *match = argv[i];

824

		cairo_bool_t invert = match[0] == '!';

825

		if (invert)

826

		    match++;

827

828

		if (runner.keyword_match) {

829

		    if (keywords != NULL && strstr (keywords, match) != NULL) {

830

			found = ! invert;

831

			break;

832

		    } else if (invert) {

833

			found = TRUE;

834

835

		} else {

836

		    /* exact match on test name */

837

		    if (strcmp (name, match) == 0) {

838

			found = ! invert;

839

			break;

840

		    } else if (invert) {

841

			found = TRUE;

842

843

844

845

846

	    if (! found) {

847

		free (name);

848

		continue;

849

850

851

852

	/* check to see if external requirements match */

853

621

	if (test->requirements != NULL) {

854

89

	    const char *requirements = test->requirements;

855

	    const char *str;

856

857

89

	    str = strstr (requirements, "slow");

858

89

	    if (str != NULL && ! runner.slow) {

859

14

		if (runner.list_only)

860

		    goto TEST_NEXT;

861

		else

862

14

		    goto TEST_SKIPPED;

863

864

865

75

	    str = strstr (requirements, "cairo");

866

75

	    if (str != NULL && ! _has_required_cairo_version (str)) {

867

		if (runner.list_only)

868

		    goto TEST_NEXT;

869

		else

870

		    goto TEST_SKIPPED;

871

872

873

874

607

	if (runner.list_only) {

875

	    printf ("%s ", name);

876

	    goto TEST_NEXT;

877

878

879

607

	_cairo_test_context_init_for_test (&ctx, &runner.base, test);

880

607

	memset (target_status, 0,

881

607

		sizeof (cairo_test_status_t) * ctx.num_targets);

882

883

607

	if (ctx.test->preamble != NULL) {

884

44

	    status = _cairo_test_runner_preamble (&runner, &ctx);

885

44

	    if (getenv ("CAIRO_TEST_UGLY_HACK_TO_IGNORE_PS_FAILURES")) {

886

		if (strcmp (ctx.test_name, "ps-eps") == 0) {

887

		    if (status == CAIRO_TEST_FAILURE) {

888

			cairo_test_log (&ctx, "Turning FAIL into XFAIL due to env\n");

889

			fprintf (stderr, "Turning FAIL into XFAIL due to env\n");

890

			runner.num_ignored_via_env++;

891

			status = CAIRO_TEST_XFAILURE;

892

		    } else {

893

			fprintf (stderr, "Test was expected to fail due to an environment variable, but did not!\n");

894

			fprintf (stderr, "Please update the corresponding CAIRO_TEST_IGNORE_* variable.\n");

895

			status = CAIRO_TEST_ERROR;

896

897

898

899

44

	    switch (status) {

900

27

	    case CAIRO_TEST_SUCCESS:

901

27

		in_preamble = TRUE;

902

27

		skipped = FALSE;

903

27

		break;

904

905

1

	    case CAIRO_TEST_XFAILURE:

906

1

		in_preamble = TRUE;

907

1

		xfailed = TRUE;

908

1

		goto TEST_DONE;

909

910

	    case CAIRO_TEST_NEW:

911

	    case CAIRO_TEST_FAILURE:

912

		runner.fails_preamble = _list_prepend (runner.fails_preamble,

913

						       test);

914

		in_preamble = TRUE;

915

		failed = TRUE;

916

		goto TEST_DONE;

917

918

	    case CAIRO_TEST_ERROR:

919

		runner.errors_preamble = _list_prepend (runner.errors_preamble,

920

							 test);

921

		in_preamble = TRUE;

922

		failed = TRUE;

923

		goto TEST_DONE;

924

925

	    case CAIRO_TEST_NO_MEMORY:

926

	    case CAIRO_TEST_CRASHED:

927

		runner.crashes_preamble = _list_prepend (runner.crashes_preamble,

928

							 test);

929

		in_preamble = TRUE;

930

		failed = TRUE;

931

		goto TEST_DONE;

932

933

16

	    case CAIRO_TEST_UNTESTED:

934

16

		goto TEST_DONE;

935

936

937

938

590

	if (ctx.test->draw == NULL)

939

16

	    goto TEST_DONE;

940

941

2296

	for (n = 0; n < ctx.num_targets; n++) {

942

	    const cairo_boilerplate_target_t *target;

943

1722

	    cairo_bool_t target_failed = FALSE,

944

1722

			 target_xfailed = FALSE,

945

1722

			 target_error = FALSE,

946

1722

			 target_crashed = FALSE,

947

1722

			 target_skipped = TRUE;

948

	    cairo_test_similar_t has_similar;

949

950

1722

	    target = ctx.targets_to_test[n];

951

952

3444

	    has_similar = runner.full_test & TEST_SIMILAR ?

953

1722

			  cairo_test_target_has_similar (&ctx, target) :

954

			  DIRECT;

955

3444

	    for (m = 0; m < runner.num_device_offsets; m++) {

956

3444

		for (k = 0; k < runner.num_device_scales; k++) {

957

1722

		    int dev_offset = m * 25;

958

1722

		    int dev_scale = k + 1;

959

		    cairo_test_similar_t similar;

960

961

3444

		    for (similar = DIRECT; similar <= has_similar; similar++) {

962

1722

			status = _cairo_test_runner_draw (&runner, &ctx, target,

963

							  similar, dev_offset, dev_scale);

964

965

1722

			if (expect_fail_due_to_env_var (&ctx, target)) {

966

377

			    if (status == CAIRO_TEST_FAILURE) {

967

366

				cairo_test_log (&ctx, "Turning FAIL into XFAIL due to env\n");

968

366

				fprintf (stderr, "Turning FAIL into XFAIL due to env\n");

969

366

				runner.num_ignored_via_env++;

970

366

				status = CAIRO_TEST_XFAILURE;

971

			    } else {

972

11

				fprintf (stderr, "Test was expected to fail due to an environment variable, but did not!\n");

973

11

				fprintf (stderr, "Please update the corresponding CAIRO_TEST_IGNORE_* variable.\n");

974

11

				status = CAIRO_TEST_ERROR;

975

976

977

1722

			if (getenv ("CAIRO_TEST_UGLY_HACK_TO_SOMETIMES_IGNORE_SCRIPT_XCB_HUGE_IMAGE_SHM")) {

978

			    if (strcmp (target->name, "script") == 0 && strcmp (ctx.test_name, "xcb-huge-image-shm") == 0) {

979

				if (status == CAIRO_TEST_FAILURE) {

980

				    fprintf (stderr, "This time the xcb-huge-image-shm test on script surface failed.\n");

981

				    cairo_test_log (&ctx, "Turning FAIL into XFAIL due to env\n");

982

				    fprintf (stderr, "Turning FAIL into XFAIL due to env\n");

983

				    runner.num_ignored_via_env++;

984

				} else {

985

				    fprintf (stderr, "This time the xcb-huge-image-shm test on script surface did not fail.\n");

986

				    cairo_test_log (&ctx, "Turning the status into XFAIL due to env\n");

987

				    fprintf (stderr, "Turning the status into XFAIL due to env\n");

988

989

				status = CAIRO_TEST_XFAILURE;

990

				fprintf (stderr, "If you are were getting one of the outcomes for some time, please update this code.\n");

991

992

993

1722

			switch (status) {

994

1311

			case CAIRO_TEST_SUCCESS:

995

1311

			    target_skipped = FALSE;

996

1311

			    break;

997

397

			case CAIRO_TEST_XFAILURE:

998

397

			    target_xfailed = TRUE;

999

397

			    break;

1000

			case CAIRO_TEST_NEW:

1001

			case CAIRO_TEST_FAILURE:

1002

			    target_failed = TRUE;

1003

			    break;

1004

11

			case CAIRO_TEST_ERROR:

1005

11

			    target_error = TRUE;

1006

11

			    break;

1007

			case CAIRO_TEST_NO_MEMORY:

1008

			case CAIRO_TEST_CRASHED:

1009

			    target_crashed = TRUE;

1010

			    break;

1011

3

			case CAIRO_TEST_UNTESTED:

1012

3

			    break;

1013

1014

1015

1016

1017

1018

1722

	    if (target_crashed) {

1019

		target_status[n] = CAIRO_TEST_CRASHED;

1020

		runner.num_crashed_per_target[n]++;

1021

		runner.crashes_per_target[n] = _list_prepend (runner.crashes_per_target[n],

1022

							      test);

1023

		crashed = TRUE;

1024

1722

	    } else if (target_error) {

1025

11

		target_status[n] = CAIRO_TEST_ERROR;

1026

11

		runner.num_error_per_target[n]++;

1027

11

		runner.errors_per_target[n] = _list_prepend (runner.errors_per_target[n],

1028

							     test);

1029

1030

11

		error = TRUE;

1031

1711

	    } else if (target_failed) {

1032

		target_status[n] = CAIRO_TEST_FAILURE;

1033

		runner.num_failed_per_target[n]++;

1034

		runner.fails_per_target[n] = _list_prepend (runner.fails_per_target[n],

1035

							    test);

1036

1037

		failed = TRUE;

1038

1711

	    } else if (target_xfailed) {

1039

397

		target_status[n] = CAIRO_TEST_XFAILURE;

1040

397

		xfailed = TRUE;

1041

1314

	    } else if (target_skipped) {

1042

3

		target_status[n] = CAIRO_TEST_UNTESTED;

1043

	    } else {

1044

1311

		target_status[n] = CAIRO_TEST_SUCCESS;

1045

1311

		skipped = FALSE;

1046

1047

1048

1049

574

  TEST_DONE:

1050

607

	cairo_test_fini (&ctx);

1051

621

  TEST_SKIPPED:

1052

621

	targets[0] = '\0';

1053

621

	if (crashed) {

1054

	    if (! in_preamble) {

1055

		len = 0;

1056

		for (n = 0 ; n < runner.base.num_targets; n++) {

1057

		    if (target_status[n] == CAIRO_TEST_CRASHED) {

1058

			if (strstr (targets,

1059

				    runner.base.targets_to_test[n]->name) == NULL)

1060

1061

			    len += snprintf (targets + len, sizeof (targets) - len,

1062

					     "%s, ",

1063

					     runner.base.targets_to_test[n]->name);

1064

1065

1066

1067

		targets[len-2] = '\0';

1068

		_log (&runner.base, "\n%s: CRASH! (%s)\n", name, targets);

1069

	    } else {

1070

		_log (&runner.base, "\n%s: CRASH!\n", name);

1071

1072

	    runner.num_crashed++;

1073

	    runner.passed = FALSE;

1074

621

	} else if (error) {

1075

6

	    if (! in_preamble) {

1076

6

		len = 0;

1077

24

		for (n = 0 ; n < runner.base.num_targets; n++) {

1078

18

		    if (target_status[n] == CAIRO_TEST_ERROR) {

1079

11

			if (strstr (targets,

1080

11

				    runner.base.targets_to_test[n]->name) == NULL)

1081

1082

6

			    len += snprintf (targets + len,

1083

					     sizeof (targets) - len,

1084

					     "%s, ",

1085

6

					     runner.base.targets_to_test[n]->name);

1086

1087

1088

1089

6

		targets[len-2] = '\0';

1090

6

		_log (&runner.base, "%s: ERROR (%s)\n", name, targets);

1091

	    } else {

1092

		_log (&runner.base, "%s: ERROR\n", name);

1093

1094

6

	    runner.num_error++;

1095

6

	    runner.passed = FALSE;

1096

615

	} else if (failed) {

1097

	    if (! in_preamble) {

1098

		len = 0;

1099

		for (n = 0 ; n < runner.base.num_targets; n++) {

1100

		    if (target_status[n] == CAIRO_TEST_FAILURE) {

1101

			if (strstr (targets,

1102

				    runner.base.targets_to_test[n]->name) == NULL)

1103

1104

			    len += snprintf (targets + len,

1105

					     sizeof (targets) - len,

1106

					     "%s, ",

1107

					     runner.base.targets_to_test[n]->name);

1108

1109

1110

1111

		targets[len-2] = '\0';

1112

		_log (&runner.base, "%s: FAIL (%s)\n", name, targets);

1113

	    } else {

1114

		_log (&runner.base, "%s: FAIL\n", name);

1115

1116

	    runner.num_failed++;

1117

	    runner.passed = FALSE;

1118

615

	} else if (xfailed) {

1119

295

	    _log (&runner.base, "%s: XFAIL\n", name);

1120

295

	    runner.num_xfailed++;

1121

320

	} else if (skipped) {

1122

31

	    _log (&runner.base, "%s: UNTESTED\n", name);

1123

31

	    runner.num_skipped++;

1124

	} else {

1125

289

	    _log (&runner.base, "%s: PASS\n", name);

1126

289

	    runner.num_passed++;

1127

1128

621

	fflush (runner.base.log_file);

1129

1130

621

  TEST_NEXT:

1131

621

	free (name);

1132

621

	if (runner.exit_on_failure && ! runner.passed)

1133

	    break;

1134

1135

1136

1137

1

    if (cairo_tests_env)

1138

	free(argv);

1139

1140

1

    if (runner.list_only) {

1141

	printf ("\n");

1142

	return CAIRO_TEST_SUCCESS;

1143

1144

1145

4

    for (n = 0 ; n < runner.base.num_targets; n++) {

1146

3

	runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]);

1147

3

	runner.errors_per_target[n] = _list_reverse (runner.errors_per_target[n]);

1148

3

	runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]);

1149

1150

1151

1

    _runner_print_results (&runner);

1152

1153

1

    _list_free (tests);

1154

1

    free (target_status);

1155

1

    return _runner_fini (&runner);

1156

1157

1158

void

1159

621

cairo_test_register (cairo_test_t *test)

1160

1161

621

    tests = _list_prepend (tests, test);

1162

621