1 /++
2   A module containing the Visual Trial reporter used to send data to the
3   Visual Studio Code plugin
4 
5   Copyright: © 2017 Szabo Bogdan
6   License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
7   Authors: Szabo Bogdan
8 +/
9 module trial.reporters.visualtrial;
10 
11 import std.conv;
12 import std..string;
13 import std.algorithm;
14 import std.stdio;
15 import std.datetime;
16 import std.exception;
17 
18 version(Have_fluent_asserts) {
19   import fluentasserts.core.base;
20   import fluentasserts.core.results;
21 }
22 
23 import trial.interfaces;
24 import trial.reporters.writer;
25 
26 enum Tokens : string {
27   beginTest = "BEGIN TEST;",
28   endTest = "END TEST;",
29   suite = "suite",
30   test = "test",
31   file = "file",
32   line = "line",
33   labels = "labels",
34   status = "status",
35   errorFile = "errorFile",
36   errorLine = "errorLine",
37   message = "message"
38 }
39 
40 /// This reporter will print the results using thr Test anything protocol version 13
41 class VisualTrialReporter : ILifecycleListener, ITestCaseLifecycleListener
42 {
43   private {
44     ReportWriter writer;
45   }
46 
47   ///
48   this()
49   {
50     writer = defaultWriter;
51   }
52 
53   ///
54   this(ReportWriter writer)
55   {
56     this.writer = writer;
57   }
58 
59   ///
60   void begin(ulong testCount) {
61     writer.writeln("", ReportWriter.Context._default);
62     writer.writeln("", ReportWriter.Context._default);
63   }
64 
65   ///
66   void update() { }
67 
68   ///
69   void end(SuiteResult[]) { }
70 
71   ///
72   void begin(string suite, ref TestResult result)
73   {
74     std.stdio.stdout.flush;
75     std.stdio.stderr.flush;
76     writer.writeln("BEGIN TEST;", ReportWriter.Context._default);
77     writer.writeln("suite:" ~ suite, ReportWriter.Context._default);
78     writer.writeln("test:" ~ result.name, ReportWriter.Context._default);
79     writer.writeln("file:" ~ result.fileName, ReportWriter.Context._default);
80     writer.writeln("line:" ~ result.line.to!string, ReportWriter.Context._default);
81     writer.writeln("labels:[" ~ result.labels.map!(a => a.toString).join(", ") ~ "]", ReportWriter.Context._default);
82     std.stdio.stdout.flush;
83     std.stdio.stderr.flush;
84   }
85 
86   ///
87   void end(string suite, ref TestResult test)
88   {
89     std.stdio.stdout.flush;
90     std.stdio.stderr.flush;
91     writer.writeln("status:" ~ test.status.to!string, ReportWriter.Context._default);
92 
93     if(test.status != TestResult.Status.success) {
94       if(test.throwable !is null) {
95         writer.writeln("errorFile:" ~ test.throwable.file, ReportWriter.Context._default);
96         writer.writeln("errorLine:" ~ test.throwable.line.to!string, ReportWriter.Context._default);
97         writer.writeln("message:" ~ test.throwable.msg.split("\n")[0], ReportWriter.Context._default);
98         writer.write("error:", ReportWriter.Context._default);
99         writer.writeln(test.throwable.toString, ReportWriter.Context._default);
100       }
101     }
102 
103     writer.writeln("END TEST;", ReportWriter.Context._default);
104 
105     std.stdio.stdout.flush;
106     std.stdio.stderr.flush;
107   }
108 }
109 
110 /// it should print "The Plan" at the beginning
111 unittest {
112   auto writer = new BufferedWriter;
113   auto reporter = new VisualTrialReporter(writer);
114   reporter.begin(10);
115 
116   writer.buffer.should.equal("\n\n");
117 }
118 
119 /// it should print the test location
120 unittest
121 {
122   auto writer = new BufferedWriter;
123   auto reporter = new VisualTrialReporter(writer);
124 
125   auto test = new TestResult("other test");
126   test.fileName = "someFile.d";
127   test.line = 100;
128   test.labels = [ Label("name", "value"), Label("name1", "value1") ];
129   test.status = TestResult.Status.success;
130 
131   reporter.begin("some suite", test);
132 
133   writer.buffer.should.equal("BEGIN TEST;\n" ~
134     "suite:some suite\n" ~
135     "test:other test\n" ~
136     "file:someFile.d\n" ~
137     "line:100\n" ~
138     `labels:[{ "name": "name", "value": "value" }, { "name": "name1", "value": "value1" }]` ~ "\n");
139 }
140 
141 /// it should print a sucess test
142 unittest
143 {
144   auto writer = new BufferedWriter;
145   auto reporter = new VisualTrialReporter(writer);
146 
147   auto test = new TestResult("other test");
148   test.fileName = "someFile.d";
149   test.line = 100;
150   test.status = TestResult.Status.success;
151 
152   reporter.end("some suite", test);
153 
154   writer.buffer.should.equal("status:success\nEND TEST;\n");
155 }
156 
157 /// it should print a failing test with a basic throwable
158 unittest
159 {
160   auto writer = new BufferedWriter;
161   auto reporter = new VisualTrialReporter(writer);
162 
163   auto test = new TestResult("other's test");
164   test.status = TestResult.Status.failure;
165   test.throwable = new Exception("Test's failure", "file.d", 42);
166 
167   reporter.end("some suite", test);
168 
169   writer.buffer.should.equal("status:failure\n" ~
170          "errorFile:file.d\n" ~
171          "errorLine:42\n" ~
172          "message:Test's failure\n" ~
173          "error:object.Exception@file.d(42): Test's failure\n" ~
174          "END TEST;\n");
175 }
176 
177 /// it should not print the YAML if the throwable is missing
178 unittest
179 {
180   auto writer = new BufferedWriter;
181   auto reporter = new VisualTrialReporter(writer);
182 
183   auto test = new TestResult("other's test");
184   test.status = TestResult.Status.failure;
185 
186   reporter.end("some suite", test);
187 
188   writer.buffer.should.equal("status:failure\nEND TEST;\n");
189 }
190 
191 /// it should print the results of a TestException
192 unittest {
193   IResult[] results = [
194     cast(IResult) new MessageResult("message"),
195     cast(IResult) new ExtraMissingResult("a", "b") ];
196 
197   auto exception = new TestException(results, "unknown", 0);
198 
199   auto writer = new BufferedWriter;
200   auto reporter = new VisualTrialReporter(writer);
201 
202   auto test = new TestResult("other's test");
203   test.status = TestResult.Status.failure;
204   test.throwable = exception;
205 
206   reporter.end("some suite", test);
207 
208   writer.buffer.should.equal("status:failure\n" ~
209          "errorFile:unknown\n" ~
210          "errorLine:0\n" ~
211          "message:message\n" ~
212          "error:fluentasserts.core.base.TestException@unknown(0): message\n\n" ~
213          "    Extra:a\n" ~
214          "  Missing:b\n\n" ~
215          "END TEST;\n");
216 }
217 
218 /// Parse the output from the visual trial reporter
219 class VisualTrialReporterParser {
220   TestResult testResult;
221   string suite;
222   bool readingTest;
223 
224   alias ResultEvent = void delegate(TestResult);
225   alias OutputEvent = void delegate(string);
226 
227   ResultEvent onResult;
228   OutputEvent onOutput;
229 
230   private {
231     bool readingErrorMessage;
232   }
233 
234   /// add a line to the parser
235   void add(string line) {
236     if(line == Tokens.beginTest) {
237       if(testResult is null) {
238         testResult = new TestResult("unknown");
239       }
240       readingTest = true;
241       testResult.begin = Clock.currTime;
242       testResult.end = Clock.currTime;
243       return;
244     }
245 
246     if(line == Tokens.endTest) {
247       enforce(testResult !is null, "The test result was not created!");
248       readingTest = false;
249       if(onResult !is null) {
250         onResult(testResult);
251       }
252       
253       readingErrorMessage = false;
254       testResult = null;
255       return;
256     }
257 
258     if(!readingTest) {
259       return;
260     }
261 
262     if(readingErrorMessage) {
263       testResult.throwable.msg ~= "\n" ~ line;
264       return;
265     }
266 
267     auto pos = line.indexOf(":");
268     
269     if(pos == -1) {
270       if(onOutput !is null) {
271         onOutput(line);
272       }
273 
274       return;
275     }
276 
277     string token = line[0 .. pos];
278     string value = line[pos+1 .. $];
279 
280     switch(token) {
281       case Tokens.suite:
282         suite = value;
283         break;
284 
285       case Tokens.test:
286         testResult.name = value;
287         break;
288 
289       case Tokens.file:
290         testResult.fileName = value;
291         break;
292 
293       case Tokens.line:
294         testResult.line = value.to!size_t;
295         break;
296 
297       case Tokens.labels:
298         testResult.labels = Label.fromJsonArray(value);
299         break;
300 
301       case Tokens.status:
302         testResult.status = value.to!(TestResult.Status);
303         break;
304 
305       case Tokens.errorFile:
306         if(testResult.throwable is null) {
307           testResult.throwable = new ParsedVisualTrialException();
308         }
309         testResult.throwable.file = value;
310 
311         break;
312 
313       case Tokens.errorLine:
314         if(testResult.throwable is null) {
315           testResult.throwable = new ParsedVisualTrialException();
316         }
317         testResult.throwable.line = value.to!size_t;
318         break;
319 
320       case Tokens.message:
321         enforce(testResult.throwable !is null, "The throwable must exist!");
322         testResult.throwable.msg = value;
323         readingErrorMessage = true;
324         break;
325 
326       default:
327         if(onOutput !is null) {
328           onOutput(line);
329         }
330     }
331   }
332 }
333 
334 /// Parse a successful test
335 unittest {
336   auto parser = new VisualTrialReporterParser();
337   parser.testResult.should.beNull;
338   auto begin = Clock.currTime;
339 
340   parser.add("BEGIN TEST;");
341   parser.testResult.should.not.beNull;
342   parser.testResult.begin.should.be.greaterThan(begin);
343   parser.testResult.end.should.be.greaterThan(begin);
344   parser.testResult.status.should.equal(TestResult.Status.created);
345 
346   parser.add("suite:suite name");
347   parser.suite.should.equal("suite name");
348 
349   parser.add("test:test name");
350   parser.testResult.name.should.equal("test name");
351 
352   parser.add("file:some file.d");
353   parser.testResult.fileName.should.equal("some file.d");
354 
355   parser.add("line:22");
356   parser.testResult.line.should.equal(22);
357 
358   parser.add(`labels:[ { "name": "name1", "value": "label1" }, { "name": "name2", "value": "label2" }]`);
359   parser.testResult.labels.should.equal([Label("name1", "label1"), Label("name2", "label2")]);
360 
361   parser.add("status:success");
362   parser.testResult.status.should.equal(TestResult.Status.success);
363 
364   parser.add("END TEST;");
365   parser.testResult.should.beNull;
366 }
367 
368 
369 /// Parse a failing test
370 unittest {
371   auto parser = new VisualTrialReporterParser();
372   parser.testResult.should.beNull;
373   auto begin = Clock.currTime;
374 
375   parser.add("BEGIN TEST;");
376 
377   parser.add("errorFile:file.d");
378   parser.add("errorLine:147");
379   parser.add("message:line1");
380   parser.add("line2");
381   parser.add("line3");
382 
383   parser.testResult.throwable.should.not.beNull;
384   parser.testResult.throwable.file.should.equal("file.d");
385   parser.testResult.throwable.line.should.equal(147);
386 
387   parser.add("END TEST;");
388   parser.testResult.should.beNull;
389 }
390 
391 /// Raise an event when the test is ended
392 unittest {
393   bool called;
394 
395   void checkResult(TestResult result) {
396     called = true;
397     result.should.not.beNull;
398   }
399 
400   auto parser = new VisualTrialReporterParser();
401   parser.onResult = &checkResult;
402 
403   parser.add("BEGIN TEST;");
404   parser.add("END TEST;");
405 
406   called.should.equal(true);
407 }
408 
409 /// It should not replace a test result that was already assigned
410 unittest {
411   auto testResult = new TestResult("");
412 
413   auto parser = new VisualTrialReporterParser();
414   parser.testResult = testResult;
415   parser.add("BEGIN TEST;");
416   parser.testResult.should.equal(testResult);
417 
418   parser.add("END TEST;");
419   parser.testResult.should.beNull;
420 }
421 
422 /// It should raise an event with unparsed lines
423 unittest {
424   bool raised;
425   auto parser = new VisualTrialReporterParser();
426 
427   void onOutput(string line) {
428     line.should.equal("some output");
429     raised = true;
430   }
431 
432   parser.onOutput = &onOutput;
433   parser.add("BEGIN TEST;");
434   parser.add("some output");
435 
436   raised.should.equal(true);
437 }
438 
439 class ParsedVisualTrialException : Exception {
440   this() { 
441     super("");
442   }
443 }