1 /++ 2 A module containing the interfaces used for extending the test runner 3 4 Copyright: © 2017 Szabo Bogdan 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Szabo Bogdan 7 +/ 8 module trial.interfaces; 9 10 import std.datetime; 11 import std.algorithm; 12 import std.array; 13 import std.functional; 14 import std.conv; 15 import std.file; 16 import std.path; 17 import std.uuid; 18 import std.exception; 19 import std.json; 20 import std.algorithm; 21 22 version(unittest) { 23 version(Have_fluent_asserts) { 24 import fluent.asserts; 25 } 26 } 27 28 /// Alias to a Test Case function type 29 alias TestCaseDelegate = void delegate() @system; 30 alias TestCaseFunction = void function() @system; 31 32 /// A Listener for the main test events 33 interface ILifecycleListener 34 { 35 /// This method is trigered when before the test start 36 void begin(ulong testCount); 37 38 /** 39 This method is triggered when you can perform some updates. 40 The frequency varries by the test executor that you choose 41 */ 42 void update(); 43 44 /// This method is trigered when your tests are ended 45 void end(SuiteResult[]); 46 } 47 48 /// Convert a Throwable to a json string 49 string toJsonString(Throwable throwable) { 50 if(throwable is null) { 51 return "{}"; 52 } 53 54 string fields; 55 56 fields ~= `"file":"` ~ throwable.file.escapeJson ~ `",`; 57 fields ~= `"line":"` ~ throwable.line.to!string.escapeJson ~ `",`; 58 fields ~= `"msg":"` ~ throwable.msg.escapeJson ~ `",`; 59 fields ~= `"info":"` ~ throwable.info.to!string.escapeJson ~ `",`; 60 fields ~= `"raw":"` ~ throwable.toString.escapeJson ~ `"`; 61 62 return "{" ~ fields ~ "}"; 63 } 64 65 /// convert a Throwable to json 66 unittest { 67 auto exception = new Exception("some message", __FILE__, 58); 68 exception.toJsonString.should.equal(`{"file":"lifecycle/trial/interfaces.d","line":"58","msg":"some message","info":"null","raw":"object.Exception@lifecycle/trial/interfaces.d(58): some message"}`); 69 } 70 71 72 /// A listener that provides test cases to be executed 73 interface ITestDiscovery { 74 /// Get the test cases from the compiled source code 75 TestCase[] getTestCases(); 76 } 77 78 /// A listener that provides test cases contained in a certain file 79 interface ITestDescribe { 80 /// Get the test cases by parsing the source code 81 TestCase[] discoverTestCases(string file); 82 } 83 84 /** 85 A Listener that can run tests. During the test execution can be used only one 86 instance of this listance. After all the tests were executed the result of all 87 three methods are concatenated and passed to `ILifecycleListener.end(SuiteResult[])` 88 */ 89 interface ITestExecutor 90 { 91 /// Called before all tests were discovered and they are ready to be executed 92 SuiteResult[] beginExecution(ref const(TestCase)[]); 93 94 /// Run a particullary test case 95 SuiteResult[] execute(ref const(TestCase)); 96 97 /// Called when there is no more test to be executed 98 SuiteResult[] endExecution(); 99 } 100 101 /// A Listener for the suite events 102 interface ISuiteLifecycleListener 103 { 104 /// Called before a suite execution 105 void begin(ref SuiteResult); 106 107 /// Called after a suite execution 108 void end(ref SuiteResult); 109 } 110 111 /// A Listener for handling attachments 112 interface IAttachmentListener 113 { 114 /// Called when an attachment is ready 115 void attach(ref const Attachment); 116 } 117 118 /// A Listener for the test case events 119 interface ITestCaseLifecycleListener 120 { 121 /// Called before a test execution 122 void begin(string suite, ref TestResult); 123 124 // Called after a test execution 125 void end(string suite, ref TestResult); 126 } 127 128 /// A Listener for the step events 129 interface IStepLifecycleListener 130 { 131 /// Called before a step begins 132 void begin(string suite, string test, ref StepResult); 133 134 /// Called after a step ended 135 void end(string suite, string test, ref StepResult); 136 } 137 138 /// A struct representing a label for test results 139 struct Label { 140 /// The label name 141 string name; 142 143 /// The label value 144 string value; 145 146 /// Convert the struct to a JSON string 147 string toString() inout { 148 return `{ "name": "` ~ name.escapeJson ~ `", "value": "` ~ value.escapeJson ~ `" }`; 149 } 150 151 /// 152 static Label[] fromJsonArray(string value) { 153 return parseJSON(value).array.map!(a => Label(a["name"].str, a["value"].str)).array; 154 } 155 156 /// 157 static Label fromJson(string value) { 158 auto parsedValue = parseJSON(value); 159 160 return Label(parsedValue["name"].str, parsedValue["value"].str); 161 } 162 } 163 164 /// Label string representation should be in Json format 165 unittest { 166 Label("name", "value").toString.should.equal(`{ "name": "name", "value": "value" }`); 167 } 168 169 /// create a label from a json object 170 unittest { 171 auto label = Label.fromJson(`{ "name": "name", "value": "value" }`); 172 173 label.name.should.equal("name"); 174 label.value.should.equal("value"); 175 } 176 177 /// create a label list from a json array 178 unittest { 179 auto labels = Label.fromJsonArray(`[{ "name": "name1", "value": "value1" }, { "name": "name2", "value": "value2" }]`); 180 181 labels.should.equal([ Label("name1", "value1"), Label("name2", "value2") ]); 182 } 183 184 /// A struct representing an attachment for test steps 185 struct Attachment { 186 /// The attachment name 187 string name; 188 189 /// The absolute path to the attachment 190 string file; 191 192 /// The file mime path 193 string mime; 194 195 /// The attachement destination. All the attached files will be copied in this folder if 196 /// it is not allready inside 197 static string destination; 198 199 /// Add a file to the current test or step 200 static Attachment fromFile(const string name, const string path, const string mime) { 201 auto fileDestination = buildPath(destination, randomUUID.toString ~ "." ~ path.baseName); 202 copy(path, fileDestination); 203 204 auto a = const Attachment(name, fileDestination, mime); 205 206 if(LifeCycleListeners.instance !is null) { 207 LifeCycleListeners.instance.attach(a); 208 } 209 210 return a; 211 } 212 213 /// Create an attachment from a string 214 static Attachment fromString(const string content) { 215 ulong index; 216 string fileDestination = buildPath(destination, randomUUID.toString ~ ".txt"); 217 fileDestination.write(content); 218 219 auto a = const Attachment("unknown", fileDestination, "text/plain"); 220 221 if(LifeCycleListeners.instance !is null) { 222 LifeCycleListeners.instance.attach(a); 223 } 224 225 return a; 226 } 227 228 /// 229 string toString() inout { 230 string fields; 231 fields ~= `"name":"` ~ name ~ `",`; 232 fields ~= `"file":"` ~ file ~ `",`; 233 fields ~= `"mime":"` ~ mime ~ `"`; 234 235 return "{" ~ fields ~ "}"; 236 } 237 } 238 239 /// Create an attachement from string 240 unittest { 241 auto a = Attachment.fromString("content"); 242 a.name.should.equal("unknown"); 243 a.mime.should.equal("text/plain"); 244 245 readText(a.file).should.equal("content"); 246 } 247 248 /// Convert an attachement to Json string 249 unittest { 250 Attachment("dub", "dub.json", "text/json").toString.should.equal( 251 `{"name":"dub","file":"dub.json","mime":"text/json"}` 252 ); 253 } 254 255 /// Represents a line of code in a certain file. 256 struct SourceLocation { 257 /// 258 string fileName; 259 260 /// 261 size_t line; 262 263 /// Converts the structure to a JSON string 264 string toString() inout { 265 return `{ "fileName": "` ~ fileName.escapeJson ~ `", "line": ` ~ line.to!string ~ ` }`; 266 } 267 } 268 269 270 /// SourceLocation string representation should be a JSON string 271 unittest { 272 SourceLocation("file.d", 10).toString.should.equal(`{ "fileName": "file.d", "line": 10 }`); 273 } 274 275 private string escapeJson(string value) { 276 return value.replace(`"`, `\"`).replace("\r", `\r`).replace("\n", `\n`).replace("\t", `\t`); 277 } 278 279 /// A test case that will be executed 280 struct TestCase 281 { 282 /** 283 The test case suite name. It can contain `.` which is treated as a 284 separator for nested suites 285 */ 286 string suiteName; 287 288 /// The test name 289 string name; 290 291 /** 292 The function that must be executed to check if the test passes or not. 293 In case of failure, an exception is thrown. 294 */ 295 TestCaseDelegate func; 296 297 /** 298 A list of labels that will be added to the final report 299 */ 300 Label[] labels; 301 302 /// The test location 303 SourceLocation location; 304 305 /// 306 this(const TestCase testCase) { 307 suiteName = testCase.suiteName.dup; 308 name = testCase.name.dup; 309 func = testCase.func; 310 location = testCase.location; 311 labels.length = testCase.labels.length; 312 313 foreach(key, val; testCase.labels) { 314 labels[key] = val; 315 } 316 } 317 318 /// 319 this(T)(string suiteName, string name, T func, Label[] labels, SourceLocation location) { 320 this(suiteName, name, func.toDelegate, labels); 321 this.location = location; 322 } 323 324 /// 325 this(string suiteName, string name, TestCaseFunction func, Label[] labels = []) { 326 this(suiteName, name, func.toDelegate, labels); 327 } 328 329 /// 330 this(string suiteName, string name, TestCaseDelegate func, Label[] labels = []) { 331 this.suiteName = suiteName; 332 this.name = name; 333 this.func = func; 334 this.labels = labels; 335 } 336 337 string toString() const { 338 string jsonRepresentation = "{ "; 339 340 jsonRepresentation ~= `"suiteName": "` ~ suiteName.escapeJson ~ `", `; 341 jsonRepresentation ~= `"name": "` ~ name.escapeJson ~ `", `; 342 jsonRepresentation ~= `"labels": [ ` ~ labels.map!(a => a.toString).join(", ") ~ ` ], `; 343 jsonRepresentation ~= `"location": ` ~ location.toString; 344 345 return jsonRepresentation ~ " }"; 346 } 347 } 348 349 /// TestCase string representation should be a JSON string 350 unittest { 351 void MockTest() {} 352 353 auto testCase = TestCase("some suite", "some name", &MockTest, [ Label("label1", "value1"), Label("label2", "value2") ]); 354 testCase.location = SourceLocation("file.d", 42); 355 356 testCase.toString.should.equal(`{ "suiteName": "some suite", "name": "some name", ` ~ 357 `"labels": [ { "name": "label1", "value": "value1" }, { "name": "label2", "value": "value2" } ], ` ~ 358 `"location": { "fileName": "file.d", "line": 42 } }`); 359 } 360 361 /// 362 TestResult toTestResult(const TestCase testCase) { 363 auto testResult = new TestResult(testCase.name.dup); 364 365 testResult.begin = Clock.currTime; 366 testResult.end = testResult.begin; 367 testResult.labels = testCase.labels.dup; 368 testResult.fileName = testCase.location.fileName; 369 testResult.line = testCase.location.line; 370 371 return testResult; 372 } 373 374 /// A suite result 375 struct SuiteResult 376 { 377 /** 378 The suite name. It can contain `.` which is treated as a 379 separator for nested suites 380 */ 381 string name; 382 383 /// when the suite started 384 SysTime begin; 385 386 /// when the suite ended 387 SysTime end; 388 389 /// the tests executed for the current suite 390 TestResult[] tests; 391 392 /// The list of attached files 393 Attachment[] attachments; 394 395 /// 396 @disable 397 this(); 398 399 /// 400 this(string name) { 401 this.name = name; 402 begin = SysTime.fromUnixTime(0); 403 end = SysTime.fromUnixTime(0); 404 } 405 406 /// 407 this(string name, SysTime begin) { 408 this.name = name; 409 this.begin = begin; 410 } 411 412 /// 413 this(string name, SysTime begin, SysTime end) { 414 this.name = name; 415 this.begin = begin; 416 this.end = end; 417 } 418 419 /// 420 this(string name, SysTime begin, SysTime end, TestResult[] tests) { 421 this.name = name; 422 this.begin = begin; 423 this.end = end; 424 this.tests = tests; 425 } 426 427 /// 428 this(string name, SysTime begin, SysTime end, TestResult[] tests, Attachment[] attachments) { 429 this.name = name; 430 this.begin = begin; 431 this.end = end; 432 this.tests = tests; 433 this.attachments = attachments; 434 } 435 436 /// Convert the struct to a json string 437 string toString() { 438 string fields; 439 fields ~= `"name":"` ~ name.escapeJson ~ `",`; 440 fields ~= `"begin":"` ~ begin.toISOExtString ~ `",`; 441 fields ~= `"end":"` ~ end.toISOExtString ~ `",`; 442 fields ~= `"tests":[` ~ tests.map!(a => a.toString).join(",") ~ `],`; 443 fields ~= `"attachments":[` ~ attachments.map!(a => a.toString).join(",") ~ `]`; 444 445 return "{" ~ fields ~ "}"; 446 } 447 } 448 449 unittest { 450 auto result = SuiteResult("suite name", 451 SysTime.fromISOExtString("2000-01-01T00:00:00Z"), 452 SysTime.fromISOExtString("2000-01-01T01:00:00Z"), 453 [ new TestResult("test name") ], 454 [ Attachment() ]); 455 456 result.toString.should.equal( 457 `{"name":"suite name","begin":"2000-01-01T00:00:00Z","end":"2000-01-01T01:00:00Z","tests":[{"name":"test name","begin":"-29227-04-19T21:11:54.5224192Z","end":"-29227-04-19T21:11:54.5224192Z","steps":[],"attachments":[],"fileName":"","line":"0","status":"created","labels":[],"throwable":{}}],"attachments":[{"name":"","file":"","mime":""}]}` 458 ); 459 } 460 461 /// A step result 462 class StepResult 463 { 464 /// The step name 465 string name; 466 467 /// When the step started 468 SysTime begin; 469 470 /// When the step ended 471 SysTime end; 472 473 /// The list of the child steps 474 StepResult[] steps; 475 476 /// The list of attached files 477 Attachment[] attachments; 478 479 this() { 480 begin = SysTime.min; 481 end = SysTime.min; 482 } 483 484 protected string fields() { 485 string result; 486 487 result ~= `"name":"` ~ name.escapeJson ~ `",`; 488 result ~= `"begin":"` ~ begin.toISOExtString ~ `",`; 489 result ~= `"end":"` ~ end.toISOExtString ~ `",`; 490 result ~= `"steps":[` ~ steps.map!(a => a.toString).join(",") ~ `],`; 491 result ~= `"attachments":[` ~ attachments.map!(a => a.toString).join(",") ~ `]`; 492 493 return result; 494 } 495 496 /// Convert the result to a json string 497 override string toString() { 498 return "{" ~ fields ~ "}"; 499 } 500 } 501 502 /// Convert a step result to a json 503 unittest { 504 auto step = new StepResult(); 505 step.name = "step name"; 506 step.begin = SysTime.fromISOExtString("2000-01-01T00:00:00Z"); 507 step.end = SysTime.fromISOExtString("2000-01-01T01:00:00Z"); 508 step.steps = [ new StepResult() ]; 509 step.attachments = [ Attachment() ]; 510 511 step.toString.should.equal(`{"name":"step name","begin":"2000-01-01T00:00:00Z","end":"2000-01-01T01:00:00Z","steps":` ~ 512 `[{"name":"","begin":"-29227-04-19T21:11:54.5224192Z","end":"-29227-04-19T21:11:54.5224192Z","steps":[],"attachments":`~ 513 `[]}],"attachments":[{"name":"","file":"","mime":""}]}`); 514 } 515 516 /// A test result 517 class TestResult : StepResult 518 { 519 /// The states that a test can have. 520 enum Status 521 { 522 /// 523 created, 524 /// 525 failure, 526 /// 527 skip, 528 /// 529 started, 530 /// 531 success, 532 /// 533 pending, 534 /// 535 unknown 536 } 537 538 /// The file that contains this test 539 string fileName; 540 541 /// The line where this test starts 542 size_t line; 543 544 /// Represents the test status 545 Status status = Status.created; 546 547 /** 548 A list of labels that will be added to the final report 549 */ 550 Label[] labels; 551 552 /** 553 The reason why a test has failed. This value must be set only if the tests has the 554 `failure` state 555 */ 556 Throwable throwable; 557 558 /// Convenience constructor that sets the test name 559 this(string name) 560 { 561 this.name = name; 562 super(); 563 } 564 565 /// Convert the result to a json string 566 override string toString() { 567 string result = fields ~ ","; 568 569 result ~= `"fileName":"` ~ fileName.escapeJson ~ `",`; 570 result ~= `"line":"` ~ line.to!string ~ `",`; 571 result ~= `"status":"` ~ status.to!string ~ `",`; 572 result ~= `"labels":[` ~ labels.map!(a => a.toString).join(",") ~ `],`; 573 result ~= `"throwable":` ~ throwable.toJsonString; 574 575 return "{" ~ result ~ "}"; 576 } 577 } 578 579 /// Attribute that marks the test as flaky. Different reporters will interpret this information 580 /// in different ways. 581 struct Flaky { 582 583 /// Returns the labels that set the test a flaky 584 static Label[] labels() { 585 return [Label("status_details", "flaky")]; 586 } 587 } 588 589 /// Attribute that links an issue to a test. Some test reporters can display links, so the value can be also 590 /// a link. 591 struct Issue { 592 593 private string name; 594 595 /// Returns the labels that set the issue label 596 Label[] labels() { 597 return [ Label("issue", name) ]; 598 } 599 } 600 601 /// Attribute that sets the feaure label 602 struct Feature { 603 private string name; 604 605 /// Returns the labels that set the feature label 606 Label[] labels() { 607 return [ Label("feature", name) ]; 608 } 609 } 610 611 /// Attribute that sets the story label 612 struct Story { 613 private string name; 614 615 /// Returns the labels that set the feature label 616 Label[] labels() { 617 return [ Label("story", name) ]; 618 } 619 } 620 621 /// Attach the readme file 622 unittest { 623 auto attachment = Attachment.fromFile("readme file", "README.md", "text/plain"); 624 625 attachment.file.exists.should.equal(true); 626 } 627 628 /// An exception that should be thrown by the pending test cases 629 class PendingTestException : Exception { 630 631 /// 632 this(string file = __FILE__, size_t line = __LINE__, Throwable next = null) { 633 super("You cannot run pending tests", file, line, next); 634 } 635 } 636 637 /// The lifecycle listeners collections. You must use this instance in order 638 /// to extend the runner. You can have as many listeners as you want. The only restriction 639 /// is for ITestExecutor, which has no sense to have more than one instance for a run 640 class LifeCycleListeners { 641 642 /// The global instange. 643 static LifeCycleListeners instance; 644 645 private { 646 ISuiteLifecycleListener[] suiteListeners; 647 ITestCaseLifecycleListener[] testListeners; 648 IStepLifecycleListener[] stepListeners; 649 ILifecycleListener[] lifecycleListeners; 650 ITestDiscovery[] testDiscoveryListeners; 651 IAttachmentListener[] attachmentListeners; 652 ITestExecutor executor; 653 654 string currentTest; 655 bool started; 656 } 657 658 @property { 659 /// Return an unique name for the current running test. If there is no test running it 660 /// will return an empty string 661 string runningTest() const nothrow { 662 return currentTest; 663 } 664 665 /// True if the tests are being executed 666 bool isRunning() { 667 return started; 668 } 669 } 670 671 /// 672 TestCase[] getTestCases() { 673 return testDiscoveryListeners.map!(a => a.getTestCases).join; 674 } 675 676 /// Add a listener to the collection 677 void add(T)(T listener) { 678 static if(!is(CommonType!(ISuiteLifecycleListener, T) == void)) { 679 suiteListeners ~= cast(ISuiteLifecycleListener) listener; 680 suiteListeners = suiteListeners.filter!(a => a !is null).array; 681 } 682 683 static if(!is(CommonType!(ITestCaseLifecycleListener, T) == void)) { 684 testListeners ~= cast(ITestCaseLifecycleListener) listener; 685 testListeners = testListeners.filter!(a => a !is null).array; 686 } 687 688 static if(!is(CommonType!(IStepLifecycleListener, T) == void)) { 689 stepListeners ~= cast(IStepLifecycleListener) listener; 690 stepListeners = stepListeners.filter!(a => a !is null).array; 691 } 692 693 static if(!is(CommonType!(ILifecycleListener, T) == void)) { 694 lifecycleListeners ~= cast(ILifecycleListener) listener; 695 lifecycleListeners = lifecycleListeners.filter!(a => a !is null).array; 696 } 697 698 static if(!is(CommonType!(ITestExecutor, T) == void)) { 699 if(cast(ITestExecutor) listener !is null) { 700 executor = cast(ITestExecutor) listener; 701 } 702 } 703 704 static if(!is(CommonType!(ITestDiscovery, T) == void)) { 705 testDiscoveryListeners ~= cast(ITestDiscovery) listener; 706 testDiscoveryListeners = testDiscoveryListeners.filter!(a => a !is null).array; 707 } 708 709 static if(!is(CommonType!(IAttachmentListener, T) == void)) { 710 attachmentListeners ~= cast(IAttachmentListener) listener; 711 attachmentListeners = attachmentListeners.filter!(a => a !is null).array; 712 } 713 } 714 715 /// Send the attachment to all listeners 716 void attach(ref const Attachment attachment) { 717 attachmentListeners.each!(a => a.attach(attachment)); 718 } 719 720 /// Send the update event to all listeners 721 void update() { 722 lifecycleListeners.each!"a.update"; 723 } 724 725 /// Send the begin run event to all listeners 726 void begin(ulong testCount) { 727 lifecycleListeners.each!(a => a.begin(testCount)); 728 } 729 730 /// Send the end runer event to all listeners 731 void end(SuiteResult[] result) { 732 lifecycleListeners.each!(a => a.end(result)); 733 } 734 735 /// Send the begin suite event to all listeners 736 void begin(ref SuiteResult suite) { 737 suiteListeners.each!(a => a.begin(suite)); 738 } 739 740 /// Send the end suite event to all listeners 741 void end(ref SuiteResult suite) { 742 suiteListeners.each!(a => a.end(suite)); 743 } 744 745 /// Send the begin test event to all listeners 746 void begin(string suite, ref TestResult test) { 747 currentTest = suite ~ "." ~ test.name; 748 testListeners.each!(a => a.begin(suite, test)); 749 } 750 751 /// Send the end test event to all listeners 752 void end(string suite, ref TestResult test) { 753 currentTest = ""; 754 testListeners.each!(a => a.end(suite, test)); 755 } 756 757 /// Send the begin step event to all listeners 758 void begin(string suite, string test, ref StepResult step) { 759 currentTest = suite ~ "." ~ test ~ "." ~ step.name; 760 stepListeners.each!(a => a.begin(suite, test, step)); 761 } 762 763 /// Send the end step event to all listeners 764 void end(string suite, string test, ref StepResult step) { 765 currentTest = ""; 766 stepListeners.each!(a => a.end(suite, test, step)); 767 } 768 769 /// Send the execute test to the executor listener 770 SuiteResult[] execute(ref const(TestCase) func) { 771 started = true; 772 scope(exit) started = false; 773 return executor.execute(func); 774 } 775 776 /// Send the begin execution with the test case list to the executor listener 777 SuiteResult[] beginExecution(ref const(TestCase)[] tests) { 778 enforce(executor !is null, "The test executor was not set."); 779 return executor.beginExecution(tests); 780 } 781 782 /// Send the end execution the executor listener 783 SuiteResult[] endExecution() { 784 return executor.endExecution(); 785 } 786 }