1 /++
2   A module containing the interfaces used for extending the test runner
3 
4   Copyright: © 2017 Szabo Bogdan
5   License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6   Authors: Szabo Bogdan
7 +/
8 module trial.interfaces;
9 
10 import std.datetime;
11 import std.algorithm;
12 import std.array;
13 import std.functional;
14 import std.conv;
15 import std.file;
16 import std.path;
17 import std.uuid;
18 import std.exception;
19 import std.json;
20 import std.algorithm;
21 
22 version(unittest) {
23   version(Have_fluent_asserts) {
24     import fluent.asserts;
25   }
26 }
27 
28 /// Alias to a Test Case function type
29 alias TestCaseDelegate = void delegate() @system;
30 alias TestCaseFunction = void function() @system;
31 
32 /// A Listener for the main test events
33 interface ILifecycleListener
34 {
35   /// This method is trigered when before the test start
36   void begin(ulong testCount);
37 
38   /**
39    This method is triggered when you can perform some updates.
40    The frequency varries by the test executor that you choose
41    */
42   void update();
43 
44   /// This method is trigered when your tests are ended
45   void end(SuiteResult[]);
46 }
47 
48 /// Convert a Throwable to a json string
49 string toJsonString(Throwable throwable) {
50   if(throwable is null) {
51     return "{}";
52   }
53 
54   string fields;
55 
56   fields ~= `"file":"` ~ throwable.file.escapeJson ~ `",`;
57   fields ~= `"line":"` ~ throwable.line.to!string.escapeJson ~ `",`;
58   fields ~= `"msg":"` ~ throwable.msg.escapeJson ~ `",`;
59   fields ~= `"info":"` ~ throwable.info.to!string.escapeJson ~ `",`;
60   fields ~= `"raw":"` ~ throwable.toString.escapeJson ~ `"`;
61 
62   return "{" ~ fields ~ "}";
63 }
64 
65 /// convert a Throwable to json
66 unittest {
67   auto exception = new Exception("some message", __FILE__, 58);
68   exception.toJsonString.should.equal(`{"file":"lifecycle/trial/interfaces.d","line":"58","msg":"some message","info":"null","raw":"object.Exception@lifecycle/trial/interfaces.d(58): some message"}`);
69 }
70 
71 
72 /// A listener that provides test cases to be executed
73 interface ITestDiscovery {
74   /// Get the test cases from the compiled source code
75   TestCase[] getTestCases();
76 }
77 
78 /// A listener that provides test cases contained in a certain file
79 interface ITestDescribe {
80   /// Get the test cases by parsing the source code
81   TestCase[] discoverTestCases(string file);
82 }
83 
84 /**
85 A Listener that can run tests. During the test execution can be used only one
86 instance of this listance. After all the tests were executed the result of all
87 three methods are concatenated and passed to `ILifecycleListener.end(SuiteResult[])`
88 */
89 interface ITestExecutor
90 {
91   /// Called before all tests were discovered and they are ready to be executed
92   SuiteResult[] beginExecution(ref const(TestCase)[]);
93 
94   /// Run a particullary test case
95   SuiteResult[] execute(ref const(TestCase));
96 
97   /// Called when there is no more test to be executed
98   SuiteResult[] endExecution();
99 }
100 
101 /// A Listener for the suite events
102 interface ISuiteLifecycleListener
103 {
104   /// Called before a suite execution
105   void begin(ref SuiteResult);
106 
107   /// Called after a suite execution
108   void end(ref SuiteResult);
109 }
110 
111 /// A Listener for handling attachments
112 interface IAttachmentListener
113 {
114   /// Called when an attachment is ready
115   void attach(ref const Attachment);
116 }
117 
118 /// A Listener for the test case events
119 interface ITestCaseLifecycleListener
120 {
121   /// Called before a test execution
122   void begin(string suite, ref TestResult);
123 
124   // Called after a test execution
125   void end(string suite, ref TestResult);
126 }
127 
128 /// A Listener for the step events
129 interface IStepLifecycleListener
130 {
131   /// Called before a step begins
132   void begin(string suite, string test, ref StepResult);
133 
134   /// Called after a step ended
135   void end(string suite, string test, ref StepResult);
136 }
137 
138 /// A struct representing a label for test results
139 struct Label {
140   /// The label name
141   string name;
142 
143   /// The label value
144   string value;
145 
146   /// Convert the struct to a JSON string
147   string toString() inout {
148     return `{ "name": "` ~ name.escapeJson ~ `", "value": "` ~ value.escapeJson ~ `" }`;
149   }
150 
151   ///
152   static Label[] fromJsonArray(string value) {
153     return parseJSON(value).array.map!(a => Label(a["name"].str, a["value"].str)).array;
154   }
155 
156   ///
157   static Label fromJson(string value) {
158     auto parsedValue = parseJSON(value);
159 
160     return Label(parsedValue["name"].str, parsedValue["value"].str);
161   }
162 }
163 
164 /// Label string representation should be in Json format
165 unittest {
166   Label("name", "value").toString.should.equal(`{ "name": "name", "value": "value" }`);
167 }
168 
169 /// create a label from a json object
170 unittest {
171   auto label = Label.fromJson(`{ "name": "name", "value": "value" }`);
172 
173   label.name.should.equal("name");
174   label.value.should.equal("value");
175 }
176 
177 /// create a label list from a json array
178 unittest {
179   auto labels = Label.fromJsonArray(`[{ "name": "name1", "value": "value1" }, { "name": "name2", "value": "value2" }]`);
180 
181   labels.should.equal([ Label("name1", "value1"), Label("name2", "value2") ]);
182 }
183 
184 /// A struct representing an attachment for test steps
185 struct Attachment {
186   /// The attachment name
187   string name;
188 
189   /// The absolute path to the attachment
190   string file;
191 
192   /// The file mime path
193   string mime;
194 
195   /// The attachement destination. All the attached files will be copied in this folder if 
196   /// it is not allready inside
197   static string destination;
198 
199   /// Add a file to the current test or step
200   static Attachment fromFile(const string name, const string path, const string mime) {
201     auto fileDestination = buildPath(destination, randomUUID.toString ~ "." ~ path.baseName);
202     copy(path, fileDestination);
203 
204     auto a = const Attachment(name, fileDestination, mime);
205 
206     if(LifeCycleListeners.instance !is null) {
207       LifeCycleListeners.instance.attach(a);
208     }
209 
210     return a;
211   }
212 
213   /// Create an attachment from a string
214   static Attachment fromString(const string content) {
215     ulong index;
216     string fileDestination = buildPath(destination, randomUUID.toString ~ ".txt");
217     fileDestination.write(content);
218 
219     auto a = const Attachment("unknown", fileDestination, "text/plain");
220 
221     if(LifeCycleListeners.instance !is null) {
222       LifeCycleListeners.instance.attach(a);
223     }
224 
225     return a;
226   }
227 
228   ///
229   string toString() inout {
230     string fields;
231     fields ~= `"name":"` ~ name ~ `",`;
232     fields ~= `"file":"` ~ file ~ `",`;
233     fields ~= `"mime":"` ~ mime ~ `"`;
234 
235     return "{" ~ fields ~ "}";
236   }
237 }
238 
239 /// Create an attachement from string
240 unittest {
241   auto a = Attachment.fromString("content");
242   a.name.should.equal("unknown");
243   a.mime.should.equal("text/plain");
244 
245   readText(a.file).should.equal("content");
246 }
247 
248 /// Convert an attachement to Json string
249 unittest {
250   Attachment("dub", "dub.json", "text/json").toString.should.equal(
251     `{"name":"dub","file":"dub.json","mime":"text/json"}`
252   );
253 }
254 
255 /// Represents a line of code in a certain file.
256 struct SourceLocation {
257   ///
258   string fileName;
259 
260   ///
261   size_t line;
262 
263   /// Converts the structure to a JSON string
264   string toString() inout {
265     return `{ "fileName": "` ~ fileName.escapeJson ~ `", "line": ` ~ line.to!string ~ ` }`;
266   }
267 }
268 
269 
270 /// SourceLocation string representation should be a JSON string
271 unittest {
272   SourceLocation("file.d", 10).toString.should.equal(`{ "fileName": "file.d", "line": 10 }`);
273 }
274 
275 private string escapeJson(string value) {
276   return value.replace(`"`, `\"`).replace("\r", `\r`).replace("\n", `\n`).replace("\t", `\t`);
277 }
278 
279 /// A test case that will be executed
280 struct TestCase
281 {
282   /**
283   The test case suite name. It can contain `.` which is treated as a
284   separator for nested suites
285   */
286   string suiteName;
287 
288   /// The test name
289   string name;
290 
291   /**
292    The function that must be executed to check if the test passes or not.
293    In case of failure, an exception is thrown.
294   */
295   TestCaseDelegate func;
296 
297   /**
298     A list of labels that will be added to the final report
299   */
300   Label[] labels;
301 
302   /// The test location
303   SourceLocation location;
304 
305   ///
306   this(const TestCase testCase) {
307     suiteName = testCase.suiteName.dup;
308     name = testCase.name.dup;
309     func = testCase.func;
310     location = testCase.location;
311     labels.length = testCase.labels.length;
312 
313     foreach(key, val; testCase.labels) {
314       labels[key] = val;
315     }
316   }
317 
318   ///
319   this(T)(string suiteName, string name, T func, Label[] labels, SourceLocation location) {
320     this(suiteName, name, func.toDelegate, labels);
321     this.location = location;
322   }
323 
324   ///
325   this(string suiteName, string name, TestCaseFunction func, Label[] labels = []) {
326     this(suiteName, name, func.toDelegate, labels);
327   }
328 
329   ///
330   this(string suiteName, string name, TestCaseDelegate func, Label[] labels = []) {
331     this.suiteName = suiteName;
332     this.name = name;
333     this.func = func;
334     this.labels = labels;
335   }
336 
337   string toString() const {
338     string jsonRepresentation = "{ ";
339 
340     jsonRepresentation ~= `"suiteName": "` ~ suiteName.escapeJson ~ `", `;
341     jsonRepresentation ~= `"name": "` ~ name.escapeJson ~ `", `;
342     jsonRepresentation ~= `"labels": [ ` ~ labels.map!(a => a.toString).join(", ") ~ ` ], `;
343     jsonRepresentation ~= `"location": ` ~ location.toString;
344 
345     return jsonRepresentation ~ " }";
346   }
347 }
348 
349 /// TestCase string representation should be a JSON string
350 unittest {
351   void MockTest() {}
352 
353   auto testCase = TestCase("some suite", "some name", &MockTest, [ Label("label1", "value1"), Label("label2", "value2") ]);
354   testCase.location = SourceLocation("file.d", 42);
355 
356   testCase.toString.should.equal(`{ "suiteName": "some suite", "name": "some name", ` ~
357     `"labels": [ { "name": "label1", "value": "value1" }, { "name": "label2", "value": "value2" } ], ` ~
358     `"location": { "fileName": "file.d", "line": 42 } }`);
359 }
360 
361 ///
362 TestResult toTestResult(const TestCase testCase) {
363   auto testResult = new TestResult(testCase.name.dup);
364 
365   testResult.begin = Clock.currTime;
366   testResult.end = testResult.begin;
367   testResult.labels = testCase.labels.dup;
368   testResult.fileName = testCase.location.fileName;
369   testResult.line = testCase.location.line;
370 
371   return testResult;
372 }
373 
374 /// A suite result
375 struct SuiteResult
376 {
377   /**
378   The suite name. It can contain `.` which is treated as a
379   separator for nested suites
380   */
381   string name;
382 
383   /// when the suite started
384   SysTime begin;
385 
386   /// when the suite ended
387   SysTime end;
388 
389   /// the tests executed for the current suite
390   TestResult[] tests;
391 
392   /// The list of attached files
393   Attachment[] attachments;
394 
395   ///
396   @disable
397   this();
398 
399   ///
400   this(string name) {
401     this.name = name;
402     begin = SysTime.fromUnixTime(0);
403     end = SysTime.fromUnixTime(0);
404   }
405 
406   ///
407   this(string name, SysTime begin) {
408     this.name = name;
409     this.begin = begin;
410   }
411 
412   ///
413   this(string name, SysTime begin, SysTime end) {
414     this.name = name;
415     this.begin = begin;
416     this.end = end;
417   }
418 
419   ///
420   this(string name, SysTime begin, SysTime end, TestResult[] tests) {
421     this.name = name;
422     this.begin = begin;
423     this.end = end;
424     this.tests = tests;
425   }
426 
427   ///
428   this(string name, SysTime begin, SysTime end, TestResult[] tests, Attachment[] attachments) {
429     this.name = name;
430     this.begin = begin;
431     this.end = end;
432     this.tests = tests;
433     this.attachments = attachments;
434   }
435 
436   /// Convert the struct to a json string
437   string toString() {
438     string fields;
439     fields ~= `"name":"` ~ name.escapeJson ~ `",`;
440     fields ~= `"begin":"` ~ begin.toISOExtString ~ `",`;
441     fields ~= `"end":"` ~ end.toISOExtString ~ `",`;
442     fields ~= `"tests":[` ~ tests.map!(a => a.toString).join(",") ~ `],`;
443     fields ~= `"attachments":[` ~ attachments.map!(a => a.toString).join(",") ~ `]`;
444 
445     return "{" ~ fields ~ "}";
446   }
447 }
448 
449 unittest {
450   auto result = SuiteResult("suite name",
451     SysTime.fromISOExtString("2000-01-01T00:00:00Z"),
452     SysTime.fromISOExtString("2000-01-01T01:00:00Z"),
453     [ new TestResult("test name") ],
454     [ Attachment() ]);
455 
456   result.toString.should.equal(
457     `{"name":"suite name","begin":"2000-01-01T00:00:00Z","end":"2000-01-01T01:00:00Z","tests":[{"name":"test name","begin":"-29227-04-19T21:11:54.5224192Z","end":"-29227-04-19T21:11:54.5224192Z","steps":[],"attachments":[],"fileName":"","line":"0","status":"created","labels":[],"throwable":{}}],"attachments":[{"name":"","file":"","mime":""}]}`
458   );
459 }
460 
461 /// A step result
462 class StepResult
463 {
464   /// The step name
465   string name;
466 
467   /// When the step started
468   SysTime begin;
469 
470   /// When the step ended
471   SysTime end;
472 
473   /// The list of the child steps
474   StepResult[] steps;
475 
476   /// The list of attached files
477   Attachment[] attachments;
478 
479   this() {
480     begin = SysTime.min;
481     end = SysTime.min;
482   }
483 
484   protected string fields() {
485     string result;
486     
487     result ~= `"name":"` ~ name.escapeJson ~ `",`;
488     result ~= `"begin":"` ~ begin.toISOExtString ~ `",`;
489     result ~= `"end":"` ~ end.toISOExtString ~ `",`;
490     result ~= `"steps":[` ~ steps.map!(a => a.toString).join(",") ~ `],`;
491     result ~= `"attachments":[` ~ attachments.map!(a => a.toString).join(",") ~ `]`;
492 
493     return result;
494   }
495 
496   /// Convert the result to a json string
497   override string toString() {
498     return "{" ~ fields ~ "}";
499   }
500 }
501 
502 /// Convert a step result to a json
503 unittest {
504   auto step = new StepResult();
505   step.name = "step name";
506   step.begin = SysTime.fromISOExtString("2000-01-01T00:00:00Z");
507   step.end = SysTime.fromISOExtString("2000-01-01T01:00:00Z");
508   step.steps = [ new StepResult() ];
509   step.attachments = [ Attachment() ];
510 
511   step.toString.should.equal(`{"name":"step name","begin":"2000-01-01T00:00:00Z","end":"2000-01-01T01:00:00Z","steps":` ~
512   `[{"name":"","begin":"-29227-04-19T21:11:54.5224192Z","end":"-29227-04-19T21:11:54.5224192Z","steps":[],"attachments":`~
513   `[]}],"attachments":[{"name":"","file":"","mime":""}]}`);
514 }
515 
516 /// A test result
517 class TestResult : StepResult
518 {
519   /// The states that a test can have.
520   enum Status
521   {
522     ///
523     created,
524     ///
525     failure,
526     ///
527     skip,
528     ///
529     started,
530     ///
531     success,
532     ///
533     pending,
534     ///
535     unknown
536   }
537 
538   /// The file that contains this test
539   string fileName;
540 
541   /// The line where this test starts
542   size_t line;
543 
544   /// Represents the test status
545   Status status = Status.created;
546 
547   /**
548     A list of labels that will be added to the final report
549   */
550   Label[] labels;
551 
552   /**
553    The reason why a test has failed. This value must be set only if the tests has the
554    `failure` state
555    */
556   Throwable throwable;
557 
558   /// Convenience constructor that sets the test name
559   this(string name)
560   {
561     this.name = name;
562     super();
563   }
564 
565   /// Convert the result to a json string
566   override string toString() {
567     string result = fields ~ ",";
568 
569     result ~= `"fileName":"` ~ fileName.escapeJson ~ `",`;
570     result ~= `"line":"` ~ line.to!string ~ `",`;
571     result ~= `"status":"` ~ status.to!string ~ `",`;
572     result ~= `"labels":[` ~ labels.map!(a => a.toString).join(",") ~ `],`;
573     result ~= `"throwable":` ~ throwable.toJsonString;
574 
575     return "{" ~ result ~ "}";
576   }
577 }
578 
579 /// Attribute that marks the test as flaky. Different reporters will interpret this information
580 /// in different ways.
581 struct Flaky {
582 
583   /// Returns the labels that set the test a flaky
584   static Label[] labels() {
585     return [Label("status_details", "flaky")];
586   }
587 }
588 
589 /// Attribute that links an issue to a test. Some test reporters can display links, so the value can be also
590 /// a link.
591 struct Issue {
592 
593   private string name;
594 
595   /// Returns the labels that set the issue label
596   Label[] labels() {
597     return [ Label("issue", name) ];
598   }
599 }
600 
601 /// Attribute that sets the feaure label
602 struct Feature {
603   private string name;
604 
605   /// Returns the labels that set the feature label
606   Label[] labels() {
607     return [ Label("feature", name) ];
608   }
609 }
610 
611 /// Attribute that sets the story label
612 struct Story {
613   private string name;
614 
615   /// Returns the labels that set the feature label
616   Label[] labels() {
617     return [ Label("story", name) ];
618   }
619 }
620 
621 /// Attach the readme file
622 unittest {
623   auto attachment = Attachment.fromFile("readme file", "README.md", "text/plain");
624 
625   attachment.file.exists.should.equal(true);
626 }
627 
628 /// An exception that should be thrown by the pending test cases
629 class PendingTestException : Exception {
630 
631   ///
632   this(string file = __FILE__, size_t line = __LINE__, Throwable next = null)  {
633     super("You cannot run pending tests", file, line, next);
634   }
635 }
636 
637 /// The lifecycle listeners collections. You must use this instance in order
638 /// to extend the runner. You can have as many listeners as you want. The only restriction
639 /// is for ITestExecutor, which has no sense to have more than one instance for a run
640 class LifeCycleListeners {
641 
642   /// The global instange.
643   static LifeCycleListeners instance;
644 
645   private {
646     ISuiteLifecycleListener[] suiteListeners;
647     ITestCaseLifecycleListener[] testListeners;
648     IStepLifecycleListener[] stepListeners;
649     ILifecycleListener[] lifecycleListeners;
650     ITestDiscovery[] testDiscoveryListeners;
651     IAttachmentListener[] attachmentListeners;
652     ITestExecutor executor;
653 
654     string currentTest;
655     bool started;
656   }
657 
658   @property {
659     /// Return an unique name for the current running test. If there is no test running it
660     /// will return an empty string
661     string runningTest() const nothrow {
662       return currentTest;
663     }
664 
665     /// True if the tests are being executed
666     bool isRunning() {
667       return started;
668     }
669   }
670 
671   ///
672   TestCase[] getTestCases() {
673     return testDiscoveryListeners.map!(a => a.getTestCases).join;
674   }
675 
676   /// Add a listener to the collection
677   void add(T)(T listener) {
678     static if(!is(CommonType!(ISuiteLifecycleListener, T) == void)) {
679       suiteListeners ~= cast(ISuiteLifecycleListener) listener;
680       suiteListeners = suiteListeners.filter!(a => a !is null).array;
681     }
682 
683     static if(!is(CommonType!(ITestCaseLifecycleListener, T) == void)) {
684       testListeners ~= cast(ITestCaseLifecycleListener) listener;
685       testListeners = testListeners.filter!(a => a !is null).array;
686     }
687 
688     static if(!is(CommonType!(IStepLifecycleListener, T) == void)) {
689       stepListeners ~= cast(IStepLifecycleListener) listener;
690       stepListeners = stepListeners.filter!(a => a !is null).array;
691     }
692 
693     static if(!is(CommonType!(ILifecycleListener, T) == void)) {
694       lifecycleListeners ~= cast(ILifecycleListener) listener;
695       lifecycleListeners = lifecycleListeners.filter!(a => a !is null).array;
696     }
697 
698     static if(!is(CommonType!(ITestExecutor, T) == void)) {
699       if(cast(ITestExecutor) listener !is null) {
700         executor = cast(ITestExecutor) listener;
701       }
702     }
703 
704     static if(!is(CommonType!(ITestDiscovery, T) == void)) {
705       testDiscoveryListeners ~= cast(ITestDiscovery) listener;
706       testDiscoveryListeners = testDiscoveryListeners.filter!(a => a !is null).array;
707     }
708 
709     static if(!is(CommonType!(IAttachmentListener, T) == void)) {
710       attachmentListeners ~= cast(IAttachmentListener) listener;
711       attachmentListeners = attachmentListeners.filter!(a => a !is null).array;
712     }
713   }
714 
715   /// Send the attachment to all listeners
716   void attach(ref const Attachment attachment) {
717     attachmentListeners.each!(a => a.attach(attachment));
718   }
719 
720   /// Send the update event to all listeners
721   void update() {
722     lifecycleListeners.each!"a.update";
723   }
724 
725   /// Send the begin run event to all listeners
726   void begin(ulong testCount) {
727     lifecycleListeners.each!(a => a.begin(testCount));
728   }
729 
730   /// Send the end runer event to all listeners
731   void end(SuiteResult[] result) {
732     lifecycleListeners.each!(a => a.end(result));
733   }
734 
735   /// Send the begin suite event to all listeners
736   void begin(ref SuiteResult suite) {
737     suiteListeners.each!(a => a.begin(suite));
738   }
739 
740   /// Send the end suite event to all listeners
741   void end(ref SuiteResult suite) {
742     suiteListeners.each!(a => a.end(suite));
743   }
744 
745   /// Send the begin test event to all listeners
746   void begin(string suite, ref TestResult test) {
747     currentTest = suite ~ "." ~ test.name;
748     testListeners.each!(a => a.begin(suite, test));
749   }
750 
751   /// Send the end test event to all listeners
752   void end(string suite, ref TestResult test) {
753     currentTest = "";
754     testListeners.each!(a => a.end(suite, test));
755   }
756 
757   /// Send the begin step event to all listeners
758   void begin(string suite, string test, ref StepResult step) {
759     currentTest = suite ~ "." ~ test ~ "." ~ step.name;
760     stepListeners.each!(a => a.begin(suite, test, step));
761   }
762 
763   /// Send the end step event to all listeners
764   void end(string suite, string test, ref StepResult step) {
765     currentTest = "";
766     stepListeners.each!(a => a.end(suite, test, step));
767   }
768 
769   /// Send the execute test to the executor listener
770   SuiteResult[] execute(ref const(TestCase) func) {
771     started = true;
772     scope(exit) started = false;
773     return executor.execute(func);
774   }
775 
776   /// Send the begin execution with the test case list to the executor listener
777   SuiteResult[] beginExecution(ref const(TestCase)[] tests) {
778     enforce(executor !is null, "The test executor was not set.");
779     return executor.beginExecution(tests);
780   }
781 
782   /// Send the end execution the executor listener
783   SuiteResult[] endExecution() {
784     return executor.endExecution();
785   }
786 }