1 /++ 2 The main runner logic. You can find here some LifeCycle logic and test runner 3 initalization 4 5 Copyright: © 2017 Szabo Bogdan 6 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 7 Authors: Szabo Bogdan 8 +/ 9 module trial.runner; 10 11 import std.stdio; 12 import std.algorithm; 13 import std.datetime; 14 import std.range; 15 import std.traits; 16 import std..string; 17 import std.conv; 18 import std.path; 19 import std.getopt; 20 import std.file; 21 import std.path; 22 import std.exception; 23 24 import trial.settings; 25 import trial.executor.single; 26 import trial.executor.parallel; 27 import trial.executor.process; 28 29 static this() { 30 if(LifeCycleListeners.instance is null) { 31 LifeCycleListeners.instance = new LifeCycleListeners; 32 } 33 } 34 35 /// setup the LifeCycle collection 36 void setupLifecycle(Settings settings) { 37 settings.artifactsLocation = settings.artifactsLocation.asAbsolutePath.array; 38 39 Attachment.destination = buildPath(settings.artifactsLocation, "attachment"); 40 41 if(!Attachment.destination.exists) { 42 Attachment.destination.mkdirRecurse; 43 } 44 45 if(LifeCycleListeners.instance is null) { 46 LifeCycleListeners.instance = new LifeCycleListeners; 47 } 48 49 settings.reporters.map!(a => a.toLower).each!(a => addReporter(a, settings)); 50 51 addExecutor(settings.executor, settings); 52 } 53 54 void addExecutor(string name, Settings settings) { 55 switch(name) { 56 case "default": 57 LifeCycleListeners.instance.add(new DefaultExecutor); 58 break; 59 case "parallel": 60 LifeCycleListeners.instance.add(new ParallelExecutor(settings.maxThreads)); 61 break; 62 case "process": 63 LifeCycleListeners.instance.add(new ProcessExecutor()); 64 break; 65 66 default: 67 if(name != "") { 68 writeln("There is no `" ~ name ~ "` executor. Using the default."); 69 } 70 71 LifeCycleListeners.instance.add(new DefaultExecutor); 72 } 73 } 74 75 /// Adds an embeded reporter listener to the LifeCycle listeners collection 76 void addReporter(string name, Settings settings) { 77 import trial.reporters.spec; 78 import trial.reporters.specprogress; 79 import trial.reporters.specsteps; 80 import trial.reporters.dotmatrix; 81 import trial.reporters.landing; 82 import trial.reporters.progress; 83 import trial.reporters.list; 84 import trial.reporters.html; 85 import trial.reporters.allure; 86 import trial.reporters.stats; 87 import trial.reporters.result; 88 import trial.reporters.xunit; 89 import trial.reporters.tap; 90 import trial.reporters.visualtrial; 91 92 switch(name) { 93 case "spec": 94 LifeCycleListeners.instance.add(new SpecReporter(settings)); 95 break; 96 97 case "spec-progress": 98 auto storage = statsFromFile(buildPath(settings.artifactsLocation, "stats.csv")); 99 LifeCycleListeners.instance.add(new SpecProgressReporter(storage)); 100 break; 101 102 case "spec-steps": 103 LifeCycleListeners.instance.add(new SpecStepsReporter(settings)); 104 break; 105 106 case "dot-matrix": 107 LifeCycleListeners.instance.add(new DotMatrixReporter(settings.glyphs.dotMatrix)); 108 break; 109 110 case "landing": 111 LifeCycleListeners.instance.add(new LandingReporter(settings.glyphs.landing)); 112 break; 113 114 case "list": 115 LifeCycleListeners.instance.add(new ListReporter(settings)); 116 break; 117 118 case "progress": 119 LifeCycleListeners.instance.add(new ProgressReporter(settings.glyphs.progress)); 120 break; 121 122 case "html": 123 LifeCycleListeners.instance.add( 124 new HtmlReporter(buildPath(settings.artifactsLocation, "result.html"), 125 settings.warningTestDuration, 126 settings.dangerTestDuration)); 127 break; 128 129 case "allure": 130 LifeCycleListeners.instance.add(new AllureReporter(buildPath(settings.artifactsLocation, "allure"))); 131 break; 132 133 case "xunit": 134 LifeCycleListeners.instance.add(new XUnitReporter(buildPath(settings.artifactsLocation, "xunit"))); 135 break; 136 137 case "result": 138 LifeCycleListeners.instance.add(new ResultReporter(settings.glyphs.result)); 139 break; 140 141 case "stats": 142 LifeCycleListeners.instance.add(new StatsReporter(buildPath(settings.artifactsLocation, "stats.csv"))); 143 break; 144 145 case "tap": 146 LifeCycleListeners.instance.add(new TapReporter); 147 break; 148 149 case "visualtrial": 150 LifeCycleListeners.instance.add(new VisualTrialReporter); 151 break; 152 153 default: 154 writeln("There is no `" ~ name ~ "` reporter"); 155 } 156 } 157 158 /// Returns an associative array of the detected tests, 159 /// where the key is the suite name and the value is the TestCase 160 const(TestCase)[][string] describeTests() { 161 return describeTests(LifeCycleListeners.instance.getTestCases); 162 } 163 164 /// Returns an associative array of the detected tests, 165 /// where the key is the suite name and the value is the TestCase 166 const(TestCase)[][string] describeTests(const(TestCase)[] tests) { 167 const(TestCase)[][string] groupedTests; 168 169 foreach(test; tests) { 170 groupedTests[test.suiteName] ~= test; 171 } 172 173 return groupedTests; 174 } 175 176 /// 177 string toJSONHierarchy(T)(const(T)[][string] items) { 178 struct Node { 179 Node[string] nodes; 180 const(T)[] values; 181 182 void add(string[] path, const(T)[] values) { 183 if(path.length == 0) { 184 this.values = values; 185 return; 186 } 187 188 if(path[0] !in nodes) { 189 nodes[path[0]] = Node(); 190 } 191 192 nodes[path[0]].add(path[1..$], values); 193 } 194 195 string toString(int spaces = 2) { 196 string prefix = leftJustify("", spaces); 197 string endPrefix = leftJustify("", spaces - 2); 198 string listValues = ""; 199 string objectValues = ""; 200 201 if(values.length > 0) { 202 listValues = values 203 .map!(a => a.toString) 204 .map!(a => prefix ~ a) 205 .join(",\n"); 206 } 207 208 if(nodes.keys.length > 0) { 209 objectValues = nodes 210 .byKeyValue 211 .map!(a => `"` ~ a.key ~ `": ` ~ a.value.toString(spaces + 2)) 212 .map!(a => prefix ~ a) 213 .join(",\n"); 214 } 215 216 217 if(listValues != "" && objectValues != "") { 218 return "{\n" ~ objectValues ~ ",\n" ~ prefix ~ "\"\": [\n" ~ listValues ~ "\n" ~ prefix ~ "]\n" ~ endPrefix ~ "}"; 219 } 220 221 if(listValues != "") { 222 return "[\n" ~ listValues ~ "\n" ~ endPrefix ~ "]"; 223 } 224 225 return "{\n" ~ objectValues ~ "\n" ~ endPrefix ~ "}"; 226 } 227 } 228 229 Node root; 230 231 foreach(key; items.keys) { 232 root.add(key.split("."), items[key]); 233 } 234 235 return root.toString; 236 } 237 238 /// convert an assoc array to JSON hierarchy 239 unittest { 240 struct Mock { 241 string val; 242 243 string toString() inout { 244 return `"` ~ val ~ `"`; 245 } 246 } 247 248 const(Mock)[][string] mocks; 249 250 mocks["a.b"] = [ Mock("val1"), Mock("val2") ]; 251 mocks["a.c"] = [ Mock("val3") ]; 252 253 auto result = mocks.toJSONHierarchy; 254 255 result.should.contain(` 256 "b": [ 257 "val1", 258 "val2" 259 ]`); 260 result.should.contain(`"c": [ 261 "val3" 262 ]`); 263 result.should.startWith(`{ 264 "a": {`); 265 result.should.endWith(` 266 ] 267 } 268 }`); 269 } 270 271 /// it should have an empty key for items that contain both values and childs 272 unittest { 273 struct Mock { 274 string val; 275 276 string toString() inout { 277 return `"` ~ val ~ `"`; 278 } 279 } 280 281 const(Mock)[][string] mocks; 282 283 mocks["a.b"] = [ Mock("val1"), Mock("val2") ]; 284 mocks["a.b.c"] = [ Mock("val3") ]; 285 286 mocks.toJSONHierarchy.should.equal(`{ 287 "a": { 288 "b": { 289 "c": [ 290 "val3" 291 ], 292 "": [ 293 "val1", 294 "val2" 295 ] 296 } 297 } 298 }`); 299 } 300 301 /// describeTests should return the tests cases serialised in json format 302 unittest { 303 void TestMock() @system { } 304 305 TestCase[] tests; 306 tests ~= TestCase("a.b", "some test", &TestMock, [ Label("some label", "label value") ]); 307 tests ~= TestCase("a.c", "other test", &TestMock); 308 309 auto result = describeTests(tests); 310 311 result.values.length.should.equal(2); 312 result.keys.should.containOnly([ "a.b", "a.c" ]); 313 result["a.b"].length.should.equal(1); 314 result["a.c"].length.should.equal(1); 315 } 316 317 /// Runs the tests and returns the results 318 auto runTests(const(TestCase)[] tests, string testName = "", string suiteName = "") { 319 setupSegmentationHandler!true(); 320 321 const(TestCase)[] filteredTests = tests; 322 323 if(testName != "") { 324 filteredTests = tests.filter!(a => a.name.indexOf(testName) != -1).array; 325 } 326 327 if(suiteName != "") { 328 filteredTests = filteredTests.filter!(a => a.suiteName.indexOf(suiteName) != -1).array; 329 } 330 331 LifeCycleListeners.instance.begin(filteredTests.length); 332 333 SuiteResult[] results = LifeCycleListeners.instance.beginExecution(filteredTests); 334 335 foreach(test; filteredTests) { 336 results ~= LifeCycleListeners.instance.execute(test); 337 } 338 339 results ~= LifeCycleListeners.instance.endExecution; 340 LifeCycleListeners.instance.end(results); 341 342 return results; 343 } 344 345 /// Check if a suite result list is a success 346 bool isSuccess(SuiteResult[] results) { 347 return results.map!(a => a.tests).joiner.map!(a => a.status).all!(a => a == TestResult.Status.success || a == TestResult.Status.pending); 348 } 349 350 version(unittest) { 351 version(Have_fluent_asserts) { 352 import fluent.asserts; 353 } 354 } 355 356 /// It should return true for an empty result 357 unittest { 358 [].isSuccess.should.equal(true); 359 } 360 361 /// It should return true if all the tests succeded 362 unittest { 363 SuiteResult[] results = [ SuiteResult("") ]; 364 results[0].tests = [ new TestResult("") ]; 365 results[0].tests[0].status = TestResult.Status.success; 366 367 results.isSuccess.should.equal(true); 368 } 369 370 /// It should return false if one the tests failed 371 unittest { 372 SuiteResult[] results = [ SuiteResult("") ]; 373 results[0].tests = [ new TestResult("") ]; 374 results[0].tests[0].status = TestResult.Status.failure; 375 376 results.isSuccess.should.equal(false); 377 } 378 379 /// It should return the name of this test 380 unittest { 381 if(LifeCycleListeners.instance is null || !LifeCycleListeners.instance.isRunning) { 382 return; 383 } 384 385 LifeCycleListeners.instance.runningTest.should.equal("trial.runner.It should return the name of this test"); 386 } 387 388 void setupSegmentationHandler(bool testRunner)() 389 { 390 import core.runtime; 391 392 // backtrace 393 version(CRuntime_Glibc) 394 import core.sys.linux.execinfo; 395 else version(OSX) 396 import core.sys.darwin.execinfo; 397 else version(FreeBSD) 398 import core.sys.freebsd.execinfo; 399 else version(NetBSD) 400 import core.sys.netbsd.execinfo; 401 else version(Windows) 402 import core.sys.windows.stacktrace; 403 else version(Solaris) 404 import core.sys.solaris.execinfo; 405 406 static if( __traits( compiles, backtrace ) ) 407 { 408 version(Posix) { 409 import core.sys.posix.signal; // segv handler 410 411 static extern (C) void unittestSegvHandler(int signum, siginfo_t* info, void* ptr ) nothrow 412 { 413 import core.stdc.stdio; 414 415 core.stdc.stdio.printf("\n\n"); 416 417 static if(testRunner) { 418 if(signum == SIGSEGV) { 419 core.stdc.stdio.printf("Got a Segmentation Fault running "); 420 } 421 422 if(signum == SIGBUS) { 423 core.stdc.stdio.printf("Got a bus error running "); 424 } 425 426 427 if(LifeCycleListeners.instance.runningTest != "") { 428 core.stdc.stdio.printf("%s\n\n", LifeCycleListeners.instance.runningTest.ptr); 429 } else { 430 core.stdc.stdio.printf("some setup step. This is probably a Trial bug. Please create an issue on github.\n\n"); 431 } 432 } else { 433 if(signum == SIGSEGV) { 434 core.stdc.stdio.printf("Got a Segmentation Fault! "); 435 } 436 437 if(signum == SIGBUS) { 438 core.stdc.stdio.printf("Got a bus error! "); 439 } 440 441 core.stdc.stdio.printf(" This is probably a Trial bug. Please create an issue on github.\n\n"); 442 } 443 444 static enum MAXFRAMES = 128; 445 void*[MAXFRAMES] callstack; 446 int numframes; 447 448 numframes = backtrace( callstack.ptr, MAXFRAMES ); 449 backtrace_symbols_fd( callstack.ptr, numframes, 2); 450 } 451 452 sigaction_t action = void; 453 (cast(byte*) &action)[0 .. action.sizeof] = 0; 454 sigfillset( &action.sa_mask ); // block other signals 455 action.sa_flags = SA_SIGINFO | SA_RESETHAND; 456 action.sa_sigaction = &unittestSegvHandler; 457 sigaction( SIGSEGV, &action, null ); 458 sigaction( SIGBUS, &action, null ); 459 } 460 } 461 }