1 /++
2   A module containing the discovery logic for classes annodated with @Test
3 
4   Copyright: © 2017 Szabo Bogdan
5   License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6   Authors: Szabo Bogdan
7 +/
8 module trial.discovery.testclass;
9 
10 import std.meta;
11 import std.traits;
12 import std.uni;
13 import std.conv;
14 import std..string;
15 import std.algorithm;
16 import std.random;
17 import std.array;
18 
19 import trial.interfaces;
20 public import trial.attributes;
21 import trial.discovery.code;
22 
23 /// A structure that stores data about the setup events(methods)
24 struct SetupEvent
25 {
26   string name;
27 
28   TestSetupAttribute setup;
29   TestCaseFunction func;
30 }
31 
32 private
33 {
34   SetupEvent[][string] setupMethods;
35   Object[string] testClassInstances;
36   size_t[string] testMethodCount;
37   size_t[string] testMethodExecuted;
38 }
39 
40 private void methodDone(string ModuleName, string ClassName)()
41 {
42   enum key = ModuleName ~ "." ~ ClassName;
43 
44   testMethodExecuted[key]++;
45 
46   if (testMethodExecuted[key] >= testMethodCount[key])
47   {
48     if (key in setupMethods)
49     {
50       foreach (setupMethod; setupMethods[key].filter!(a => a.setup.afterAll))
51       {
52         setupMethod.func();
53       }
54     }
55 
56     testClassInstances.remove(key);
57   }
58 }
59 
60 private auto getTestClassInstance(string ModuleName, string ClassName)()
61 {
62   mixin(`import ` ~ ModuleName ~ `;`);
63   enum key = ModuleName ~ "." ~ ClassName;
64 
65   if (key !in testClassInstances)
66   {
67     mixin(`auto instance = new ` ~ ClassName ~ `();`);
68 
69     testClassInstances[key] = instance;
70     testMethodExecuted[key] = 0;
71 
72     if (key in setupMethods)
73     {
74       foreach (setupMethod; setupMethods[key].filter!(a => a.setup.beforeAll))
75       {
76         setupMethod.func();
77       }
78     }
79   }
80 
81   mixin(`return cast(` ~ key ~ `) testClassInstances[key];`);
82 }
83 
84 /// The default test discovery looks for unit test sections and groups them by module
85 class TestClassDiscovery : ITestDiscovery
86 {
87   private TestCase[] list;
88 
89   /// Returns all the test cases that were found in the modules
90   /// added with `addModule`
91   TestCase[] getTestCases()
92   {
93     return list;
94   }
95 
96   TestCase[] discoverTestCases(string file)
97   {
98     TestCase[] testCases = [];
99 
100     version (Have_fluent_asserts)
101       version (Have_libdparse)
102       {
103         import fluentasserts.core.results;
104         import std.stdio;
105 
106         auto tokens = fileToDTokens(file);
107 
108         void noTest()
109         {
110           assert(false, "you can not run this test");
111         }
112 
113         auto iterator = TokenIterator(tokens);
114         auto moduleName = iterator.skipUntilType("module").skipOne.readUntilType(";").strip;
115 
116         string lastName;
117         string lastSuite;
118 
119         DLangAttribute[] attributes;
120         int blockIndex;
121 
122         foreach (token; iterator)
123         {
124           auto type = str(token.type);
125 
126           if (type == "version")
127           {
128             iterator.skipUntilType(")");
129           }
130 
131           if (type == "unittest")
132           {
133             iterator.skipNextBlock;
134           }
135 
136           if (type == "class")
137           {
138             iterator.skipOne;
139             auto dlangClass = iterator.readClass;
140 
141             lastSuite = moduleName ~ "." ~ dlangClass.name;
142 
143             testCases ~= dlangClass.functions.filter!(a =>
144               a.hasAttribute("Test"))
145               .map!(a => TestCase(lastSuite, a.testName, &noTest, [], SourceLocation(file, a.getAttribute("Test").line)))
146               .array;
147           }
148         }
149       }
150 
151     return testCases;
152   }
153 
154   /// Add tests from a certain module
155   void addModule(string file, string moduleName)()
156   {
157     discover!moduleName;
158   }
159 
160   private
161   {
162     void discover(string ModuleName)()
163     {
164       mixin("static import " ~ ModuleName ~ ";");
165       enum classList = classMembers!(ModuleName);
166 
167       foreach (className; classList)
168       {
169         mixin("alias CurrentClass = " ~ ModuleName ~ "." ~ className ~ ";");
170 
171         static if(__traits(getProtection, CurrentClass) == "public") {
172           enum members = __traits(allMembers, CurrentClass);
173           foreach (member; members)
174           {
175             static if(__traits(hasMember, CurrentClass, member)) {
176               static if (isSetupMember!(ModuleName, className, member))
177               {
178                 enum setup = getSetup!(ModuleName, className, member);
179                 enum key = ModuleName ~ "." ~ className;
180 
181                 auto exists = key in setupMethods
182                   && !setupMethods[key].filter!(a => a.name == member).empty;
183 
184                 if (!exists)
185                 {
186                   setupMethods[key] ~= SetupEvent(member, setup, ({
187                       mixin(`auto instance = new ` ~ className ~ `();`);
188                       mixin(`instance.` ~ member ~ `;`);
189                     }));
190                 }
191               }
192             }
193           }
194         }
195       }
196 
197       foreach (className; classList)
198       {
199         enum key = ModuleName ~ "." ~ className;
200         mixin("alias CurrentClass = " ~ key ~ ";");
201         enum protection = __traits(getProtection, CurrentClass);
202 
203         static if(protection == "public") {
204           enum members = __traits(allMembers, CurrentClass);
205           testMethodCount[key] = 0;
206 
207           foreach (member; members)
208           {
209             enum memberProtection = __traits(getProtection, member);
210 
211             static if(memberProtection == "public") {
212               static if (isTestMember!(ModuleName, className, member))
213               {
214                 testMethodCount[key]++;
215                 string testName = getTestName!(ModuleName, className, member);
216 
217                 auto testCase = TestCase(ModuleName ~ "." ~ className, testName, ({
218                     auto instance = getTestClassInstance!(ModuleName, className);
219 
220                     enum key = ModuleName ~ "." ~ className;
221 
222                     if (key in setupMethods)
223                     {
224                       foreach (setupMethod; setupMethods[key].filter!(a => a.setup.beforeEach))
225                       {
226                         setupMethod.func();
227                       }
228                     }
229 
230                     mixin(`instance.` ~ member ~ `;`);
231 
232                     if (key in setupMethods)
233                     {
234                       foreach (setupMethod; setupMethods[key].filter!(a => a.setup.afterEach))
235                       {
236                         setupMethod.func();
237                       }
238                     }
239 
240                     methodDone!(ModuleName, className);
241                   }), []);
242 
243                 testCase.location = getTestLocation!(ModuleName, className, member);
244 
245                 list ~= testCase;
246               }
247             }
248           }
249         }
250       }
251     }
252   }
253 }
254 
255 ///
256 string getTestName(string ModuleName, string className, string member)()
257 {
258   mixin("static import " ~ ModuleName ~ ";");
259   mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "."
260       ~ className ~ "." ~ member ~ ");");
261 
262   enum testAttributes = testAttributes!attributes;
263 
264   string name;
265 
266   foreach (attribute; attributes)
267   {
268     static if (is(typeof(attribute) == string))
269     {
270       name = attribute;
271     }
272   }
273 
274   if (name.length == 0)
275   {
276     return member.camelToSentence;
277   }
278   else
279   {
280     return name;
281   }
282 }
283 
284 ///
285 SourceLocation getTestLocation(string ModuleName, string className, string member)()
286 {
287   mixin("static import " ~ ModuleName ~ ";");
288   mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "."
289       ~ className ~ "." ~ member ~ ");");
290 
291   enum testAttributes = testAttributes!attributes;
292 
293   return SourceLocation(testAttributes[0].file, testAttributes[0].line);
294 }
295 
296 ///
297 auto getSetup(string ModuleName, string className, string member)()
298 {
299   mixin("static import " ~ ModuleName ~ ";");
300   mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "."
301       ~ className ~ "." ~ member ~ ");");
302 
303   return setupAttributes!attributes[0];
304 }
305 
306 ///
307 bool isTestMember(string ModuleName, string className, string member)()
308 {
309   mixin("static import " ~ ModuleName ~ ";");
310 
311   static if(__traits(hasMember, mixin(ModuleName ~ "." ~ className), member)) {
312     static if(__traits(compiles, isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member)))) {
313       static if(isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member))) {
314           static if(__traits(compiles, mixin("__traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ")"))) {
315             mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ");");
316               return testAttributes!attributes.length > 0;
317             } else {
318               return false;
319             }
320         } else {
321           return false;
322         }
323     } else {
324       return false;
325     }
326   } else {
327     return false;
328   }
329 }
330 
331 ///
332 bool isSetupMember(string ModuleName, string className, string member)()
333 {
334   mixin("static import " ~ ModuleName ~ ";");
335 
336   static if(__traits(hasMember, mixin(ModuleName ~ "." ~ className), member)) {
337     static if(__traits(compiles, isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member)))) {
338       static if(isCallable!(mixin(ModuleName ~ "." ~ className ~ "." ~ member))) {
339           static if(__traits(compiles, mixin("__traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ")"))) {
340             mixin("enum attributes = __traits(getAttributes, " ~ ModuleName ~ "." ~ className ~ "." ~ member ~ ");");
341               return setupAttributes!attributes.length > 0;
342             } else {
343               return false;
344             }
345         } else {
346           return false;
347         }
348     } else {
349       return false;
350     }
351   } else {
352     return false;
353   }
354 }
355 
356 ///
357 template isClass(string moduleName)
358 {
359   template isModuleClass(string name) {
360     mixin("
361       static import " ~ moduleName ~ ";
362 
363       static if (is(" ~ moduleName ~ "." ~ name ~ " == class))
364         enum bool isModuleClass = true;
365       else
366         enum bool isModuleClass = false;");
367   }
368 
369   alias isClass = isModuleClass;
370 }
371 
372 ///
373 template isTestAttribute(alias Attribute)
374 {
375   import trial.attributes;
376 
377   static if (!is(CommonType!(Attribute, TestAttribute) == void))
378   {
379     enum bool isTestAttribute = true;
380   }
381   else
382   {
383     enum bool isTestAttribute = false;
384   }
385 }
386 
387 ///
388 template isRightParameter(string parameterName)
389 {
390   template isRightParameter(alias Attribute)
391   {
392     enum isRightParameter = Attribute.parameterName == parameterName;
393   }
394 }
395 
396 ///
397 template isSetupAttribute(alias Attribute)
398 {
399   static if (!is(CommonType!(Attribute, TestSetupAttribute) == void))
400   {
401     enum bool isSetupAttribute = true;
402   }
403   else
404   {
405     enum bool isSetupAttribute = false;
406   }
407 }
408 
409 ///
410 template isValueProvider(alias Attribute)
411 {
412   static if (__traits(hasMember, Attribute, "provide") && __traits(hasMember,
413       Attribute, "parameterName"))
414   {
415     enum bool isValueProvider = true;
416   }
417   else
418   {
419     enum bool isValueProvider = false;
420   }
421 }
422 
423 ///
424 template extractClasses(string moduleName, members...)
425 {
426   mixin("static import " ~ moduleName ~ ";");
427   alias Filter!(isClass!moduleName, members) extractClasses;
428 }
429 
430 ///
431 template extractValueProviders(Elements...)
432 {
433   alias Filter!(isValueProvider, Elements) extractValueProviders;
434 }
435 
436 ///
437 template testAttributes(attributes...)
438 {
439   alias Filter!(isTestAttribute, attributes) testAttributes;
440 }
441 
442 ///
443 template setupAttributes(attributes...)
444 {
445   alias Filter!(isSetupAttribute, attributes) setupAttributes;
446 }
447 
448 ///
449 template classMembers(string moduleName)
450 {
451   mixin("static import " ~ moduleName ~ ";");
452   mixin("alias extractClasses!(moduleName, __traits(allMembers, " ~ moduleName ~ ")) classMembers;");
453 }
454 
455 version (unittest)
456 {
457   version(Have_fluent_asserts): 
458   import trial.attributes;
459   import fluent.asserts;
460 
461   class SomeTestSuite
462   {
463     static string lastTest;
464 
465     @Test()
466     void aSimpleTest()
467     {
468       lastTest = "a simple test";
469     }
470   }
471 
472   class OtherTestSuite
473   {
474     static string[] order;
475 
476     @BeforeEach()
477     void beforeEach()
478     {
479       order ~= "before each";
480     }
481 
482     @AfterEach()
483     void afterEach()
484     {
485       order ~= "after each";
486     }
487 
488     @BeforeAll()
489     void beforeAll()
490     {
491       order ~= "before all";
492     }
493 
494     @AfterAll()
495     void afterAll()
496     {
497       order ~= "after all";
498     }
499 
500     @Test()
501     @("Some other name")
502     void aCustomTest()
503     {
504       order ~= "a custom test";
505     }
506   }
507 }
508 
509 /// TestClassDiscovery should find the Test Suite class
510 unittest
511 {
512   auto discovery = new TestClassDiscovery();
513   discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`);
514 
515   auto testCases = discovery.getTestCases;
516 
517   testCases.length.should.equal(2);
518   testCases[0].suiteName.should.equal(`trial.discovery.testclass.SomeTestSuite`);
519   testCases[0].name.should.equal(`A simple test`);
520 
521   testCases[1].suiteName.should.equal(`trial.discovery.testclass.OtherTestSuite`);
522   testCases[1].name.should.equal(`Some other name`);
523 }
524 
525 /// discoverTestCases should find the Test Suite class
526 unittest
527 {
528   auto testDiscovery = new TestClassDiscovery;
529 
530   auto testCases = testDiscovery.discoverTestCases(__FILE__);
531 
532   testCases.length.should.equal(2);
533   testCases[0].suiteName.should.equal(`trial.discovery.testclass.SomeTestSuite`);
534   testCases[0].name.should.equal(`A simple test`);
535 
536   testCases[1].suiteName.should.equal(`trial.discovery.testclass.OtherTestSuite`);
537   testCases[1].name.should.equal(`Some other name`);
538 }
539 
540 /// TestClassDiscovery should execute tests from a Test Suite class
541 unittest
542 {
543   scope (exit)
544   {
545     SomeTestSuite.lastTest = "";
546   }
547 
548   auto discovery = new TestClassDiscovery();
549   discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`);
550 
551   auto test = discovery.getTestCases.filter!(a => a.suiteName == `trial.discovery.testclass.SomeTestSuite`)
552     .filter!(a => a.name == `A simple test`).front;
553 
554   test.func();
555 
556   SomeTestSuite.lastTest.should.equal("a simple test");
557 }
558 
559 /// TestClassDiscovery should execute the before and after methods tests from a Test Suite class
560 unittest
561 {
562   scope (exit)
563   {
564     OtherTestSuite.order = [];
565   }
566 
567   auto discovery = new TestClassDiscovery();
568   discovery.addModule!(`lifecycle/trial/discovery/testclass.d`, `trial.discovery.testclass`);
569 
570   auto test = discovery.getTestCases.filter!(a => a.suiteName == `trial.discovery.testclass.OtherTestSuite`)
571     .filter!(a => a.name == `Some other name`).front;
572 
573   test.func();
574 
575   OtherTestSuite.order.should.equal(["before all", "before each",
576       "a custom test", "after each", "after all"]);
577 }
578 
579 private string generateRandomParameters(alias T, int index)() pure nothrow
580 {
581   alias paramTypes = std.traits.Parameters!T;
582   enum params = ParameterIdentifierTuple!T;
583   alias providers = Filter!(isRightParameter!(params[index].stringof[1 .. $ - 1]),
584       extractValueProviders!(__traits(getAttributes, T)));
585 
586   enum provider = "Filter!(isRightParameter!(" ~ params[index].stringof
587       ~ "), extractValueProviders!(__traits(getAttributes, T)))";
588 
589   static if (providers.length > 0)
590   {
591     immutable string definition = "auto param_" ~ params[index] ~ " = "
592       ~ provider ~ "[0]().provide; ";
593   }
594   else
595   {
596     immutable string definition = "auto param_" ~ params[index] ~ " = uniform!"
597       ~ paramTypes[index].stringof ~ "(); ";
598   }
599 
600   static if (index == 0)
601   {
602     return definition;
603   }
604   else
605   {
606     return definition ~ generateRandomParameters!(T, index - 1);
607   }
608 }
609 
610 private string generateMethodParameters(alias T, int size)()
611 {
612   enum params = ParameterIdentifierTuple!T;
613 
614   static if (size == 0)
615   {
616     return "";
617   }
618   else static if (size == 1)
619   {
620     return "param_" ~ params[0];
621   }
622   else
623   {
624     return generateMethodParameters!(T, size - 1) ~ ", param_" ~ params[size - 1];
625   }
626 }
627 
628 /// Call a method using the right data provders
629 void methodCaller(alias T, U)(U func)
630 {
631   enum parameterCount = arity!T;
632 
633   mixin(generateRandomParameters!(T, parameterCount - 1));
634   mixin("func(" ~ generateMethodParameters!(T, parameterCount) ~ ");");
635 }
636 
637 /// methodCaller should call the method with random numeric values
638 unittest
639 {
640   class TestClass
641   {
642     static int usedIntValue = 0;
643     static ulong usedUlongValue = 0;
644 
645     void randomMethod(int value, ulong other)
646     {
647       usedIntValue = value;
648       usedUlongValue = other;
649     }
650   }
651 
652   auto instance = new TestClass;
653 
654   methodCaller!(instance.randomMethod)(&instance.randomMethod);
655 
656   TestClass.usedIntValue.should.not.equal(0);
657   TestClass.usedUlongValue.should.not.equal(0);
658 }
659 
660 struct ValueProvider(string name, alias T)
661 {
662   immutable static string parameterName = name;
663 
664   auto provide()
665   {
666     return T();
667   }
668 }
669 
670 auto For(string name, alias T)()
671 {
672   return ValueProvider!(name, T)();
673 }
674 
675 version (unittest)
676 {
677   auto someCustomFunction()
678   {
679     return 6;
680   }
681 }
682 
683 /// methodCaller should call the method with custom random generators
684 unittest
685 {
686   class TestClass
687   {
688     static int usedIntValue = 0;
689     static ulong usedUlongValue = 0;
690 
691     @For!("value", { return 5; }) @For!("other", { return someCustomFunction(); }) void randomMethod(int value,
692         ulong other)
693     {
694       usedIntValue = value;
695       usedUlongValue = other;
696     }
697   }
698 
699   auto instance = new TestClass;
700 
701   methodCaller!(instance.randomMethod)(&instance.randomMethod);
702 
703   TestClass.usedIntValue.should.equal(5);
704   TestClass.usedUlongValue.should.equal(6);
705 }
706 
707 /// discoverTestCases should find the same tests like testCases
708 unittest
709 {
710   auto discovery = new TestClassDiscovery;
711   discovery.addModule!(__FILE__, `trial.discovery.testclass`);
712 
713   discovery
714     .discoverTestCases(__FILE__)
715     .map!(a => a.toString)
716     .join("\n")
717     .should.equal(
718       discovery.getTestCases
719       .sort!((a, b) => a.location.line < b.location.line)
720       .map!(a => a.toString)
721       .join("\n"));
722 }