1 /++
2   The main runner logic. You can find here some LifeCycle logic and test runner
3   initalization
4 
5   Copyright: © 2017 Szabo Bogdan
6   License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
7   Authors: Szabo Bogdan
8 +/
9 module trial.runner;
10 
11 import std.stdio;
12 import std.algorithm;
13 import std.datetime;
14 import std.range;
15 import std.traits;
16 import std..string;
17 import std.conv;
18 import std.path;
19 import std.getopt;
20 import std.file;
21 import std.path;
22 import std.exception;
23 
24 import trial.settings;
25 import trial.executor.single;
26 import trial.executor.parallel;
27 import trial.executor.process;
28 
29 static this() {
30   if(LifeCycleListeners.instance is null) {
31     LifeCycleListeners.instance = new LifeCycleListeners;
32   }
33 }
34 
35 /// setup the LifeCycle collection
36 void setupLifecycle(Settings settings) {
37   settings.artifactsLocation = settings.artifactsLocation.asAbsolutePath.array;
38 
39   Attachment.destination = buildPath(settings.artifactsLocation, "attachment");
40   
41   if(!Attachment.destination.exists) {
42     Attachment.destination.mkdirRecurse;
43   }
44 
45   if(LifeCycleListeners.instance is null) {
46     LifeCycleListeners.instance = new LifeCycleListeners;
47   }
48 
49   settings.reporters.map!(a => a.toLower).each!(a => addReporter(a, settings));
50 
51   addExecutor(settings.executor, settings);
52 }
53 
54 void addExecutor(string name, Settings settings) {
55   switch(name) {
56       case "default":
57         LifeCycleListeners.instance.add(new DefaultExecutor);
58         break;
59       case "parallel":
60         LifeCycleListeners.instance.add(new ParallelExecutor(settings.maxThreads));
61         break;
62       case "process":
63         LifeCycleListeners.instance.add(new ProcessExecutor());
64         break;
65       
66       default:
67         if(name != "") {
68           writeln("There is no `" ~ name ~ "` executor. Using the default.");
69         }
70 
71         LifeCycleListeners.instance.add(new DefaultExecutor);
72   }
73 }
74 
75 /// Adds an embeded reporter listener to the LifeCycle listeners collection
76 void addReporter(string name, Settings settings) {
77     import trial.reporters.spec;
78     import trial.reporters.specprogress;
79     import trial.reporters.specsteps;
80     import trial.reporters.dotmatrix;
81     import trial.reporters.landing;
82     import trial.reporters.progress;
83     import trial.reporters.list;
84     import trial.reporters.html;
85     import trial.reporters.allure;
86     import trial.reporters.stats;
87     import trial.reporters.result;
88     import trial.reporters.xunit;
89     import trial.reporters.tap;
90     import trial.reporters.visualtrial;
91 
92     switch(name) {
93       case "spec":
94         LifeCycleListeners.instance.add(new SpecReporter(settings));
95         break;
96 
97       case "spec-progress":
98         auto storage = statsFromFile(buildPath(settings.artifactsLocation, "stats.csv"));
99         LifeCycleListeners.instance.add(new SpecProgressReporter(storage));
100         break;
101 
102       case "spec-steps":
103         LifeCycleListeners.instance.add(new SpecStepsReporter(settings));
104         break;
105 
106       case "dot-matrix":
107         LifeCycleListeners.instance.add(new DotMatrixReporter(settings.glyphs.dotMatrix));
108         break;
109 
110       case "landing":
111         LifeCycleListeners.instance.add(new LandingReporter(settings.glyphs.landing));
112         break;
113 
114       case "list":
115         LifeCycleListeners.instance.add(new ListReporter(settings));
116         break;
117 
118       case "progress":
119         LifeCycleListeners.instance.add(new ProgressReporter(settings.glyphs.progress));
120         break;
121 
122       case "html":
123         LifeCycleListeners.instance.add(
124           new HtmlReporter(buildPath(settings.artifactsLocation, "result.html"), 
125           settings.warningTestDuration, 
126           settings.dangerTestDuration));
127         break;
128 
129       case "allure":
130         LifeCycleListeners.instance.add(new AllureReporter(buildPath(settings.artifactsLocation, "allure")));
131         break;
132 
133       case "xunit":
134         LifeCycleListeners.instance.add(new XUnitReporter(buildPath(settings.artifactsLocation, "xunit")));
135         break;
136 
137       case "result":
138         LifeCycleListeners.instance.add(new ResultReporter(settings.glyphs.result));
139         break;
140 
141       case "stats":
142         LifeCycleListeners.instance.add(new StatsReporter(buildPath(settings.artifactsLocation, "stats.csv")));
143         break;
144 
145       case "tap":
146         LifeCycleListeners.instance.add(new TapReporter);
147         break;
148 
149       case "visualtrial":
150         LifeCycleListeners.instance.add(new VisualTrialReporter);
151         break;
152 
153       default:
154         writeln("There is no `" ~ name ~ "` reporter");
155     }
156 }
157 
158 /// Returns an associative array of the detected tests,
159 /// where the key is the suite name and the value is the TestCase
160 const(TestCase)[][string] describeTests() {
161   return describeTests(LifeCycleListeners.instance.getTestCases);
162 }
163 
164 /// Returns an associative array of the detected tests,
165 /// where the key is the suite name and the value is the TestCase
166 const(TestCase)[][string] describeTests(const(TestCase)[] tests) {
167   const(TestCase)[][string] groupedTests;
168 
169   foreach(test; tests) {
170     groupedTests[test.suiteName] ~= test;
171   }
172 
173   return groupedTests;
174 }
175 
176 ///
177 string toJSONHierarchy(T)(const(T)[][string] items) {
178   struct Node {
179     Node[string] nodes;
180     const(T)[] values;
181 
182     void add(string[] path, const(T)[] values) {
183       if(path.length == 0) {
184         this.values = values;
185         return;
186       }
187 
188       if(path[0] !in nodes) {
189         nodes[path[0]] = Node();
190       }
191 
192       nodes[path[0]].add(path[1..$], values);
193     }
194 
195     string toString(int spaces = 2) {
196       string prefix = leftJustify("", spaces);
197       string endPrefix = leftJustify("", spaces - 2);
198       string listValues = "";
199       string objectValues = "";
200 
201       if(values.length > 0) {
202         listValues = values
203           .map!(a => a.toString)
204           .map!(a => prefix ~ a)
205           .join(",\n");
206       }
207 
208       if(nodes.keys.length > 0) {
209         objectValues = nodes
210               .byKeyValue
211               .map!(a => `"` ~ a.key ~ `": ` ~ a.value.toString(spaces + 2))
212               .map!(a => prefix ~ a)
213               .join(",\n");
214       }
215 
216 
217       if(listValues != "" && objectValues != "") {
218         return "{\n" ~ objectValues ~ ",\n" ~ prefix ~ "\"\": [\n" ~ listValues ~ "\n" ~ prefix ~ "]\n" ~ endPrefix ~ "}";
219       }
220 
221       if(listValues != "") {
222         return "[\n" ~ listValues ~ "\n" ~ endPrefix ~ "]";
223       }
224 
225       return "{\n" ~ objectValues ~ "\n" ~ endPrefix ~ "}";
226     }
227   }
228 
229   Node root;
230 
231   foreach(key; items.keys) {
232     root.add(key.split("."), items[key]);
233   }
234 
235   return root.toString;
236 }
237 
238 /// convert an assoc array to JSON hierarchy
239 unittest {
240   struct Mock {
241     string val;
242 
243     string toString() inout {
244       return `"` ~ val ~ `"`;
245     }
246   }
247 
248   const(Mock)[][string] mocks;
249 
250   mocks["a.b"] = [ Mock("val1"), Mock("val2") ];
251   mocks["a.c"] = [ Mock("val3") ];
252 
253   auto result = mocks.toJSONHierarchy;
254   
255   result.should.contain(`
256     "b": [
257       "val1",
258       "val2"
259     ]`);
260   result.should.contain(`"c": [
261       "val3"
262     ]`);
263   result.should.startWith(`{
264   "a": {`);
265   result.should.endWith(`
266     ]
267   }
268 }`);
269 }
270 
271 /// it should have an empty key for items that contain both values and childs
272 unittest {
273   struct Mock {
274     string val;
275 
276     string toString() inout {
277       return `"` ~ val ~ `"`;
278     }
279   }
280 
281   const(Mock)[][string] mocks;
282 
283   mocks["a.b"] = [ Mock("val1"), Mock("val2") ];
284   mocks["a.b.c"] = [ Mock("val3") ];
285 
286   mocks.toJSONHierarchy.should.equal(`{
287   "a": {
288     "b": {
289       "c": [
290         "val3"
291       ],
292       "": [
293       "val1",
294       "val2"
295       ]
296     }
297   }
298 }`);
299 }
300 
301 /// describeTests should return the tests cases serialised in json format
302 unittest {
303   void TestMock() @system { }
304 
305   TestCase[] tests;
306   tests ~= TestCase("a.b", "some test", &TestMock, [ Label("some label", "label value") ]);
307   tests ~= TestCase("a.c", "other test", &TestMock);
308 
309   auto result = describeTests(tests);
310 
311   result.values.length.should.equal(2);
312   result.keys.should.containOnly([ "a.b", "a.c" ]);
313   result["a.b"].length.should.equal(1);
314   result["a.c"].length.should.equal(1);
315 }
316 
317 /// Runs the tests and returns the results
318 auto runTests(const(TestCase)[] tests, string testName = "", string suiteName = "") {
319   setupSegmentationHandler!true();
320 
321   const(TestCase)[] filteredTests = tests;
322 
323   if(testName != "") {
324     filteredTests = tests.filter!(a => a.name.indexOf(testName) != -1).array;
325   }
326 
327   if(suiteName != "") {
328     filteredTests = filteredTests.filter!(a => a.suiteName.indexOf(suiteName) != -1).array;
329   }
330 
331   LifeCycleListeners.instance.begin(filteredTests.length);
332 
333   SuiteResult[] results = LifeCycleListeners.instance.beginExecution(filteredTests);
334 
335   foreach(test; filteredTests) {
336     results ~= LifeCycleListeners.instance.execute(test);
337   }
338 
339   results ~= LifeCycleListeners.instance.endExecution;
340   LifeCycleListeners.instance.end(results);
341 
342   return results;
343 }
344 
345 /// Check if a suite result list is a success
346 bool isSuccess(SuiteResult[] results) {
347   return results.map!(a => a.tests).joiner.map!(a => a.status).all!(a => a == TestResult.Status.success || a == TestResult.Status.pending);
348 }
349 
350 version(unittest) {
351   version(Have_fluent_asserts) {
352     import fluent.asserts;
353   }
354 }
355 
356 /// It should return true for an empty result
357 unittest {
358   [].isSuccess.should.equal(true);
359 }
360 
361 /// It should return true if all the tests succeded
362 unittest {
363   SuiteResult[] results = [ SuiteResult("") ];
364   results[0].tests = [ new TestResult("") ];
365   results[0].tests[0].status = TestResult.Status.success;
366 
367   results.isSuccess.should.equal(true);
368 }
369 
370 /// It should return false if one the tests failed
371 unittest {
372   SuiteResult[] results = [ SuiteResult("") ];
373   results[0].tests = [ new TestResult("") ];
374   results[0].tests[0].status = TestResult.Status.failure;
375 
376   results.isSuccess.should.equal(false);
377 }
378 
379 /// It should return the name of this test
380 unittest {
381   if(LifeCycleListeners.instance is null || !LifeCycleListeners.instance.isRunning) {
382     return;
383   }
384 
385   LifeCycleListeners.instance.runningTest.should.equal("trial.runner.It should return the name of this test");
386 }
387 
388 void setupSegmentationHandler(bool testRunner)()
389 {
390   import core.runtime;
391 
392   // backtrace
393   version(CRuntime_Glibc)
394     import core.sys.linux.execinfo;
395   else version(OSX)
396     import core.sys.darwin.execinfo;
397   else version(FreeBSD)
398     import core.sys.freebsd.execinfo;
399   else version(NetBSD)
400     import core.sys.netbsd.execinfo;
401   else version(Windows)
402     import core.sys.windows.stacktrace;
403   else version(Solaris)
404     import core.sys.solaris.execinfo;
405 
406   static if( __traits( compiles, backtrace ) )
407   {
408     version(Posix) {
409       import core.sys.posix.signal; // segv handler
410 
411       static extern (C) void unittestSegvHandler(int signum, siginfo_t* info, void* ptr ) nothrow
412       {
413         import core.stdc.stdio;
414 
415         core.stdc.stdio.printf("\n\n");
416 
417         static if(testRunner) {
418           if(signum == SIGSEGV) {
419             core.stdc.stdio.printf("Got a Segmentation Fault running ");
420           }
421 
422           if(signum == SIGBUS) {
423             core.stdc.stdio.printf("Got a bus error running ");
424           }
425 
426 
427           if(LifeCycleListeners.instance.runningTest != "") {
428             core.stdc.stdio.printf("%s\n\n", LifeCycleListeners.instance.runningTest.ptr);
429           } else {
430             core.stdc.stdio.printf("some setup step. This is probably a Trial bug. Please create an issue on github.\n\n");
431           }
432         } else {
433           if(signum == SIGSEGV) {
434             core.stdc.stdio.printf("Got a Segmentation Fault! ");
435           }
436 
437           if(signum == SIGBUS) {
438             core.stdc.stdio.printf("Got a bus error! ");
439           }
440 
441           core.stdc.stdio.printf(" This is probably a Trial bug. Please create an issue on github.\n\n");
442         }
443 
444         static enum MAXFRAMES = 128;
445         void*[MAXFRAMES]  callstack;
446         int               numframes;
447 
448         numframes = backtrace( callstack.ptr, MAXFRAMES );
449         backtrace_symbols_fd( callstack.ptr, numframes, 2);
450       }
451 
452       sigaction_t action = void;
453       (cast(byte*) &action)[0 .. action.sizeof] = 0;
454       sigfillset( &action.sa_mask ); // block other signals
455       action.sa_flags = SA_SIGINFO | SA_RESETHAND;
456       action.sa_sigaction = &unittestSegvHandler;
457       sigaction( SIGSEGV, &action, null );
458       sigaction( SIGBUS, &action, null );
459     }
460   }
461 }