1 /++ 2 A module containing the discovery logic for classes annodated with @Test 3 4 Copyright: © 2017 Szabo Bogdan 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Szabo Bogdan 7 +/ 8 module trial.discovery.testclass; 9 10 import std.meta; 11 import std.traits; 12 import std.uni; 13 import std.conv; 14 import std..string; 15 import std.algorithm; 16 import std.random; 17 import std.array; 18 19 import trial.interfaces; 20 public import trial.attributes; 21 import trial.discovery.code; 22 23 /// A structure that stores data about the setup events(methods) 24 struct SetupEvent 25 { 26 string name; 27 28 TestSetupAttribute setup; 29 TestCaseFunction func; 30 } 31 32 private 33 { 34 SetupEvent[][string] setupMethods; 35 Object[string] testClassInstances; 36 size_t[string] testMethodCount; 37 size_t[string] testMethodExecuted; 38 } 39 40 private void methodDone(string ModuleName, string ClassName)() 41 { 42 enum key = ModuleName ~ "." ~ ClassName; 43 44 testMethodExecuted[key]++; 45 46 if (testMethodExecuted[key] >= testMethodCount[key]) 47 { 48 if (key in setupMethods) 49 { 50 foreach (setupMethod; setupMethods[key].filter!(a => a.setup.afterAll)) 51 { 52 setupMethod.func(); 53 } 54 } 55 56 testClassInstances.remove(key); 57 } 58 } 59 60 private auto getTestClassInstance(string ModuleName, string ClassName)() 61 { 62 mixin(`import ` ~ ModuleName ~ `;`); 63 enum key = ModuleName ~ "." ~ ClassName; 64 65 if (key !in testClassInstances) 66 { 67 mixin(`auto instance = new ` ~ ClassName ~ `();`); 68 69 testClassInstances[key] = instance; 70 testMethodExecuted[key] = 0; 71 72 if (key in setupMethods) 73 { 74 foreach (setupMethod; setupMethods[key].filter!(a => a.setup.beforeAll)) 75 { 76 setupMethod.func(); 77 } 78 } 79 } 80 81 mixin(`return cast(` ~ key ~ `) testClassInstances[key];`); 82 } 83 84 /// The default test discovery looks for unit test sections and groups them by module 85 class TestClassDiscovery : ITestDiscovery 86 { 87 private TestCase[] list; 88 89 /// Returns all the test cases that were found in the modules 90 /// added with `addModule` 91 TestCase[] getTestCases() 92 { 93 return list; 94 } 95 96 TestCase[] discoverTestCases(string file) 97 { 98 TestCase[] testCases = []; 99 100 version (Have_fluent_asserts) 101 version (Have_libdparse) 102 { 103 import fluentasserts.core.results; 104 import std.stdio; 105 106 auto tokens = fileToDTokens(file); 107 108 void noTest() 109 { 110 assert(false, "you can not run this test"); 111 } 112 113 auto iterator = TokenIterator(tokens); 114 auto moduleName = iterator.skipUntilType("module").skipOne.readUntilType(";").strip; 115 116 string lastName; 117 string lastSuite; 118 119 DLangAttribute[] attributes; 120 int blockIndex; 121 122 foreach (token; iterator) 123 { 124 auto type = str(token.type); 125 126 if (type == "version") 127 { 128 iterator.skipUntilType(")"); 129 } 130 131 if (type == "unittest") 132 { 133 iterator.skipNextBlock; 134 } 135 136 if (type == "class") 137 { 138 iterator.skipOne; 139 auto dlangClass = iterator.readClass; 140 141 lastSuite = moduleName ~ "." ~ dlangClass.name; 142 143 testCases ~= dlangClass.functions.filter!(a => 144 a.hasAttribute("Test")) 145 .map!(a => TestCase(lastSuite, a.testName, &noTest, [], SourceLocation(file, a.getAttribute("Test").line))) 146 .array; 147 } 148 } 149 } 150 151 return testCases; 152 } 153 154 /// Add tests from a certain module 155 void addModule(string file, string moduleName)() 156 { 157 discover!moduleName; 158 } 159 160 private 161 { 162 void discover(string ModuleName)() 163 { 164 mixin("static import " ~ ModuleName ~ ";"); 165 enum classList = classMembers!(ModuleName); 166 167 foreach (className; classList) 168 { 169 mixin("alias CurrentClass = " ~ ModuleName ~ "." ~ className ~ ";"); 170 171 static if(__traits(getProtection, CurrentClass) == "public") { 172 enum members = __traits(allMembers, CurrentClass); 173 foreach (member; members) 174 { 175 static if(__traits(hasMember, CurrentClass, member)) { 176 static if (isSetupMember!(ModuleName, className, member)) 177 { 178 enum setup = getSetup!(ModuleName, className, member); 179 enum key = ModuleName ~ "." ~ className; 180 181 auto exists = key in setupMethods 182 && !setupMethods[key].filter!(a => a.name == member).empty; 183 184 if (!exists) 185 { 186 setupMethods[key] ~= SetupEvent(member, setup, ({ 187 mixin(`auto instance = new ` ~ className ~ `();`); 188 mixin(`instance.` ~ member ~ `;`); 189 })); 190 } 191 } 192 } 193 } 194 } 195 } 196 197 foreach (className; classList) 198 { 199 enum key = ModuleName ~ "." ~ className; 200 mixin("alias CurrentClass = " ~ key ~ ";"); 201 enum protection = __traits(getProtection, CurrentClass); 202 203 static if(protection == "public") { 204 enum members = __traits(allMembers, CurrentClass); 205 testMethodCount[key] = 0; 206 207 foreach (member; members) 208 { 209 enum memberProtection = __traits(getProtection, member); 210 211 static if(memberProtection == "public") { 212 static if (isTestMember!(ModuleName, className, member)) 213 { 214 testMethodCount[key]++; 215 string testName = getTestName!(ModuleName, className, member); 216 217 auto testCase = TestCase(ModuleName ~ "." ~ className, testName, ({ 218 auto instance = getTestClassInstance!(ModuleName, className); 219 220 enum key = ModuleName ~ "." ~ className; 221 222 if (key in setupMethods) 223 { 224 foreach (setupMethod; setupMethods[key].filter!(a => a.setup.beforeEach)) 225 { 226 setupMethod.func(); 227 } 228 } 229 230 mixin(`instance.` ~ member ~ `;`); 231 232 if (key in setupMethods) 233 { 234 foreach (setupMethod; setupMethods[key].filter!(a => a.setup.afterEach)) 235 { 236 setupMethod.func(); 237 } 238 } 239 240 methodDone!(ModuleName, className); 241 }), []); 242 243 testCase.location = getTestLocation!(ModuleName, className, member); 244 245 list ~= testCase; 246 } 247 } 248 } 249 } 250 } 251 } 252 } 253 } 254 255 /// 256 string getTestName(string ModuleName, string className, string member)() 257 { 258 mixin("static import " ~ ModuleName ~ ";"); 259 mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." 260 ~ className ~ "." ~ member ~ ");"); 261 262 enum testAttributes = testAttributes!attributes; 263 264 string name; 265 266 foreach (attribute; attributes) 267 { 268 static if (is(typeof(attribute) == string)) 269 { 270 name = attribute; 271 } 272 } 273 274 if (name.length == 0) 275 { 276 return member.camelToSentence; 277 } 278 else 279 { 280 return name; 281 } 282 } 283 284 /// 285 SourceLocation getTestLocation(string ModuleName, string className, string member)() 286 { 287 mixin("static import " ~ ModuleName ~ ";"); 288 mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." 289 ~ className ~ "." ~ member ~ ");"); 290 291 enum testAttributes = testAttributes!attributes; 292 293 return SourceLocation(testAttributes[0].file, testAttributes[0].line); 294 } 295 296 /// 297 auto getSetup(string ModuleName, string className, string member)() 298 { 299 mixin("static import " ~ ModuleName ~ ";"); 300 mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." 301 ~ className ~ "." ~ member ~ ");"); 302 303 return setupAttributes!attributes[0]; 304 } 305 306 /// 307 bool isTestMember(string ModuleName, string className, string member)() 308 { 309 mixin("static import " ~ ModuleName ~ ";"); 310 311 static if(__traits(hasMember, mixin(ModuleName ~ "." ~ className), member)) { 312 static if(__traits(compiles, isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member)))) { 313 static if(isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member))) { 314 static if(__traits(compiles, mixin("__traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ")"))) { 315 mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ");"); 316 return testAttributes!attributes.length > 0; 317 } else { 318 return false; 319 } 320 } else { 321 return false; 322 } 323 } else { 324 return false; 325 } 326 } else { 327 return false; 328 } 329 } 330 331 /// 332 bool isSetupMember(string ModuleName, string className, string member)() 333 { 334 mixin("static import " ~ ModuleName ~ ";"); 335 336 static if(__traits(hasMember, mixin(ModuleName ~ "." ~ className), member)) { 337 static if(__traits(compiles, isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member)))) { 338 static if(isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member))) { 339 static if(__traits(compiles, mixin("__traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ")"))) { 340 mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ");"); 341 return setupAttributes!attributes.length > 0; 342 } else { 343 return false; 344 } 345 } else { 346 return false; 347 } 348 } else { 349 return false; 350 } 351 } else { 352 return false; 353 } 354 } 355 356 /// 357 template isClass(string moduleName) 358 { 359 template isModuleClass(string name) { 360 mixin(" 361 static import " ~ moduleName ~ "; 362 363 static if (is(" ~ moduleName ~ "." ~ name ~ " == class)) 364 enum bool isModuleClass = true; 365 else 366 enum bool isModuleClass = false;"); 367 } 368 369 alias isClass = isModuleClass; 370 } 371 372 /// 373 template isTestAttribute(alias Attribute) 374 { 375 import trial.attributes; 376 377 static if (!is(CommonType!(Attribute, TestAttribute) == void)) 378 { 379 enum bool isTestAttribute = true; 380 } 381 else 382 { 383 enum bool isTestAttribute = false; 384 } 385 } 386 387 /// 388 template isRightParameter(string parameterName) 389 { 390 template isRightParameter(alias Attribute) 391 { 392 enum isRightParameter = Attribute.parameterName == parameterName; 393 } 394 } 395 396 /// 397 template isSetupAttribute(alias Attribute) 398 { 399 static if (!is(CommonType!(Attribute, TestSetupAttribute) == void)) 400 { 401 enum bool isSetupAttribute = true; 402 } 403 else 404 { 405 enum bool isSetupAttribute = false; 406 } 407 } 408 409 /// 410 template isValueProvider(alias Attribute) 411 { 412 static if (__traits(hasMember, Attribute, "provide") && __traits(hasMember, 413 Attribute, "parameterName")) 414 { 415 enum bool isValueProvider = true; 416 } 417 else 418 { 419 enum bool isValueProvider = false; 420 } 421 } 422 423 /// 424 template extractClasses(string moduleName, members...) 425 { 426 mixin("static import " ~ moduleName ~ ";"); 427 alias Filter!(isClass!moduleName, members) extractClasses; 428 } 429 430 /// 431 template extractValueProviders(Elements...) 432 { 433 alias Filter!(isValueProvider, Elements) extractValueProviders; 434 } 435 436 /// 437 template testAttributes(attributes...) 438 { 439 alias Filter!(isTestAttribute, attributes) testAttributes; 440 } 441 442 /// 443 template setupAttributes(attributes...) 444 { 445 alias Filter!(isSetupAttribute, attributes) setupAttributes; 446 } 447 448 /// 449 template classMembers(string moduleName) 450 { 451 mixin("static import " ~ moduleName ~ ";"); 452 mixin("alias extractClasses!(moduleName, __traits(allMembers, " ~ moduleName ~ ")) classMembers;"); 453 } 454 455 version (unittest) 456 { 457 version(Have_fluent_asserts): 458 import trial.attributes; 459 import fluent.asserts; 460 461 class SomeTestSuite 462 { 463 static string lastTest; 464 465 @Test() 466 void aSimpleTest() 467 { 468 lastTest = "a simple test"; 469 } 470 } 471 472 class OtherTestSuite 473 { 474 static string[] order; 475 476 @BeforeEach() 477 void beforeEach() 478 { 479 order ~= "before each"; 480 } 481 482 @AfterEach() 483 void afterEach() 484 { 485 order ~= "after each"; 486 } 487 488 @BeforeAll() 489 void beforeAll() 490 { 491 order ~= "before all"; 492 } 493 494 @AfterAll() 495 void afterAll() 496 { 497 order ~= "after all"; 498 } 499 500 @Test() 501 @("Some other name") 502 void aCustomTest() 503 { 504 order ~= "a custom test"; 505 } 506 } 507 } 508 509 /// TestClassDiscovery should find the Test Suite class 510 unittest 511 { 512 auto discovery = new TestClassDiscovery(); 513 discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`); 514 515 auto testCases = discovery.getTestCases; 516 517 testCases.length.should.equal(2); 518 testCases[0].suiteName.should.equal(`trial.discovery.testclass.SomeTestSuite`); 519 testCases[0].name.should.equal(`A simple test`); 520 521 testCases[1].suiteName.should.equal(`trial.discovery.testclass.OtherTestSuite`); 522 testCases[1].name.should.equal(`Some other name`); 523 } 524 525 /// discoverTestCases should find the Test Suite class 526 unittest 527 { 528 auto testDiscovery = new TestClassDiscovery; 529 530 auto testCases = testDiscovery.discoverTestCases(__FILE__); 531 532 testCases.length.should.equal(2); 533 testCases[0].suiteName.should.equal(`trial.discovery.testclass.SomeTestSuite`); 534 testCases[0].name.should.equal(`A simple test`); 535 536 testCases[1].suiteName.should.equal(`trial.discovery.testclass.OtherTestSuite`); 537 testCases[1].name.should.equal(`Some other name`); 538 } 539 540 /// TestClassDiscovery should execute tests from a Test Suite class 541 unittest 542 { 543 scope (exit) 544 { 545 SomeTestSuite.lastTest = ""; 546 } 547 548 auto discovery = new TestClassDiscovery(); 549 discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`); 550 551 auto test = discovery.getTestCases.filter!(a => a.suiteName == `trial.discovery.testclass.SomeTestSuite`) 552 .filter!(a => a.name == `A simple test`).front; 553 554 test.func(); 555 556 SomeTestSuite.lastTest.should.equal("a simple test"); 557 } 558 559 /// TestClassDiscovery should execute the before and after methods tests from a Test Suite class 560 unittest 561 { 562 scope (exit) 563 { 564 OtherTestSuite.order = []; 565 } 566 567 auto discovery = new TestClassDiscovery(); 568 discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`); 569 570 auto test = discovery.getTestCases.filter!(a => a.suiteName == `trial.discovery.testclass.OtherTestSuite`) 571 .filter!(a => a.name == `Some other name`).front; 572 573 test.func(); 574 575 OtherTestSuite.order.should.equal(["before all", "before each", 576 "a custom test", "after each", "after all"]); 577 } 578 579 private string generateRandomParameters(alias T, int index)() pure nothrow 580 { 581 alias paramTypes = std.traits.Parameters!T; 582 enum params = ParameterIdentifierTuple!T; 583 alias providers = Filter!(isRightParameter!(params[index].stringof[1 .. $ - 1]), 584 extractValueProviders!(__traits(getAttributes, T))); 585 586 enum provider = "Filter!(isRightParameter!(" ~ params[index].stringof 587 ~ "), extractValueProviders!(__traits(getAttributes, T)))"; 588 589 static if (providers.length > 0) 590 { 591 immutable string definition = "auto param_" ~ params[index] ~ " = " 592 ~ provider ~ "[0]().provide; "; 593 } 594 else 595 { 596 immutable string definition = "auto param_" ~ params[index] ~ " = uniform!" 597 ~ paramTypes[index].stringof ~ "(); "; 598 } 599 600 static if (index == 0) 601 { 602 return definition; 603 } 604 else 605 { 606 return definition ~ generateRandomParameters!(T, index - 1); 607 } 608 } 609 610 private string generateMethodParameters(alias T, int size)() 611 { 612 enum params = ParameterIdentifierTuple!T; 613 614 static if (size == 0) 615 { 616 return ""; 617 } 618 else static if (size == 1) 619 { 620 return "param_" ~ params[0]; 621 } 622 else 623 { 624 return generateMethodParameters!(T, size - 1) ~ ", param_" ~ params[size - 1]; 625 } 626 } 627 628 /// Call a method using the right data provders 629 void methodCaller(alias T, U)(U func) 630 { 631 enum parameterCount = arity!T; 632 633 mixin(generateRandomParameters!(T, parameterCount - 1)); 634 mixin("func(" ~ generateMethodParameters!(T, parameterCount) ~ ");"); 635 } 636 637 /// methodCaller should call the method with random numeric values 638 unittest 639 { 640 class TestClass 641 { 642 static int usedIntValue = 0; 643 static ulong usedUlongValue = 0; 644 645 void randomMethod(int value, ulong other) 646 { 647 usedIntValue = value; 648 usedUlongValue = other; 649 } 650 } 651 652 auto instance = new TestClass; 653 654 methodCaller!(instance.randomMethod)(&instance.randomMethod); 655 656 TestClass.usedIntValue.should.not.equal(0); 657 TestClass.usedUlongValue.should.not.equal(0); 658 } 659 660 struct ValueProvider(string name, alias T) 661 { 662 immutable static string parameterName = name; 663 664 auto provide() 665 { 666 return T(); 667 } 668 } 669 670 auto For(string name, alias T)() 671 { 672 return ValueProvider!(name, T)(); 673 } 674 675 version (unittest) 676 { 677 auto someCustomFunction() 678 { 679 return 6; 680 } 681 } 682 683 /// methodCaller should call the method with custom random generators 684 unittest 685 { 686 class TestClass 687 { 688 static int usedIntValue = 0; 689 static ulong usedUlongValue = 0; 690 691 @For!("value", { return 5; }) @For!("other", { return someCustomFunction(); }) void randomMethod(int value, 692 ulong other) 693 { 694 usedIntValue = value; 695 usedUlongValue = other; 696 } 697 } 698 699 auto instance = new TestClass; 700 701 methodCaller!(instance.randomMethod)(&instance.randomMethod); 702 703 TestClass.usedIntValue.should.equal(5); 704 TestClass.usedUlongValue.should.equal(6); 705 } 706 707 /// discoverTestCases should find the same tests like testCases 708 unittest 709 { 710 auto discovery = new TestClassDiscovery; 711 discovery.addModule!(__FILE__, `trial.discovery.testclass`); 712 713 discovery 714 .discoverTestCases(__FILE__) 715 .map!(a => a.toString) 716 .join("\n") 717 .should.equal( 718 discovery.getTestCases 719 .sort!((a, b) => a.location.line < b.location.line) 720 .map!(a => a.toString) 721 .join("\n")); 722 }