1 /++
2   A module containing the AllureReporter
3 
4   Copyright: © 2017 Szabo Bogdan
5   License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6   Authors: Szabo Bogdan
7 +/
8 module trial.reporters.allure;
9 
10 import std.stdio;
11 import std.array;
12 import std.conv;
13 import std.datetime;
14 import std..string;
15 import std.algorithm;
16 import std.file;
17 import std.path;
18 import std.uuid;
19 import std.range;
20 
21 import trial.interfaces;
22 import trial.reporters.writer;
23 
24 private string escape(string data) {
25   string escapedData = data.dup;
26 
27   escapedData = escapedData.replace(`&`, `&`);
28   escapedData = escapedData.replace(`"`, `"`);
29   escapedData = escapedData.replace(`'`, `'`);
30   escapedData = escapedData.replace(`<`, `&lt;`);
31   escapedData = escapedData.replace(`>`, `&gt;`);
32 
33   return escapedData;
34 }
35 
36 /// The Allure reporter creates a xml containing the test results, the steps
37 /// and the attachments. http://allure.qatools.ru/
38 class AllureReporter : ILifecycleListener
39 {
40   private {
41     immutable string destination;
42   }
43 
44   this(string destination) {
45     this.destination = destination;
46   }
47 
48   void begin(ulong testCount) {
49     if(exists(destination)) {
50       std.file.rmdirRecurse(destination);
51     }
52   }
53 
54   void update() {}
55 
56   void end(SuiteResult[] result)
57   {
58     if(!exists(destination)) {
59       destination.mkdirRecurse;
60     }
61 
62     foreach(item; result) {
63       string uuid = randomUUID.toString;
64       string xml = AllureSuiteXml(destination, item, uuid).toString;
65 
66       std.file.write(buildPath(destination,  uuid ~ "-testsuite.xml"), xml);
67     }
68   }
69 }
70 
71 struct AllureSuiteXml {
72   /// The suite result
73   SuiteResult result;
74 
75   /// The suite id
76   string uuid;
77 
78   /// The allure version
79   const string allureVersion = "1.5.2";
80 
81   private {
82     immutable string destination;
83   }
84 
85   this(const string destination, SuiteResult result, string uuid) {
86     this.destination = destination;
87     this.result = result;
88     this.uuid = uuid;
89   }
90 
91   /// Converts the suiteResult to a xml string
92   string toString() {
93     auto epoch = SysTime.fromUnixTime(0);
94     string tests = result.tests.map!(a => AllureTestXml(destination, a, uuid).toString).array.join("\n");
95 
96     if(tests != "") {
97       tests = "\n" ~ tests;
98     }
99 
100     auto xml = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
101 <ns2:test-suite start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" version="` ~ this.allureVersion ~ `" xmlns:ns2="urn:model.allure.qatools.yandex.ru">
102     <name>` ~ result.name.escape ~ `</name>
103     <title>` ~ result.name.escape ~ `</title>
104     <test-cases>`
105      ~ tests ~ `
106     </test-cases>
107 `;
108 
109     if(result.attachments.length > 0) {
110       xml ~= "    <attachments>\n";
111       xml ~= result.attachments.map!(a => AllureAttachmentXml(destination, a, 6, uuid)).map!(a => a.toString).array.join('\n') ~ "\n";
112       xml ~= "    </attachments>\n";
113     }
114 
115     xml ~= `    <labels>
116         <label name="framework" value="Trial"/>
117         <label name="language" value="D"/>
118     </labels>
119 </ns2:test-suite>`;
120 
121     return xml;
122   }
123 }
124 
125 version(unittest) {
126   import fluent.asserts;
127 }
128 
129 @("AllureSuiteXml should transform an empty suite")
130 unittest
131 {
132   auto epoch = SysTime.fromUnixTime(0);
133   auto result = SuiteResult("Test Suite");
134   result.begin = Clock.currTime;
135 
136   TestResult test = new TestResult("Test");
137   test.begin = Clock.currTime;
138   test.end = Clock.currTime;
139   test.status = TestResult.Status.success;
140 
141   result.end = Clock.currTime;
142 
143   result.tests = [ test ];
144 
145   auto allure = AllureSuiteXml("allure", result, "");
146 
147   allure.toString.should.equal(`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
148 <ns2:test-suite start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" version="1.5.2" xmlns:ns2="urn:model.allure.qatools.yandex.ru">
149     <name>` ~ result.name ~ `</name>
150     <title>` ~ result.name ~ `</title>
151     <test-cases>
152         <test-case start="` ~ (test.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
153             <name>Test</name>
154         </test-case>
155     </test-cases>
156     <labels>
157         <label name="framework" value="Trial"/>
158         <label name="language" value="D"/>
159     </labels>
160 </ns2:test-suite>`);
161 }
162 
163 @("AllureSuiteXml should transform a suite with a success test")
164 unittest
165 {
166   auto epoch = SysTime.fromUnixTime(0);
167   auto result = SuiteResult("Test Suite");
168   result.begin = Clock.currTime;
169   result.end = Clock.currTime;
170 
171   auto allure = AllureSuiteXml("allure", result, "");
172 
173   allure.toString.should.equal(`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
174 <ns2:test-suite start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" version="1.5.2" xmlns:ns2="urn:model.allure.qatools.yandex.ru">
175     <name>` ~ result.name ~ `</name>
176     <title>` ~ result.name ~ `</title>
177     <test-cases>
178     </test-cases>
179     <labels>
180         <label name="framework" value="Trial"/>
181         <label name="language" value="D"/>
182     </labels>
183 </ns2:test-suite>`);
184 }
185 
186 
187 /// AllureSuiteXml should add the attachments
188 unittest
189 {
190   string resource = buildPath(getcwd(), "some_text.txt");
191   std.file.write(resource, "");
192 
193   auto uuid = randomUUID.toString;
194 
195   scope(exit) {
196     remove(resource);
197     remove("allure/" ~ uuid ~ "/title.0.some_text.txt");
198   }
199 
200   auto epoch = SysTime.fromUnixTime(0);
201 
202   auto result = SuiteResult("Test Suite");
203   result.begin = Clock.currTime;
204   result.end = Clock.currTime;
205   result.attachments = [ Attachment("title", resource, "plain/text") ];
206 
207   auto allure = AllureSuiteXml("allure", result, uuid);
208 
209   allure.toString.should.equal(
210  `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
211 <ns2:test-suite start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" version="1.5.2" xmlns:ns2="urn:model.allure.qatools.yandex.ru">
212     <name>` ~ result.name ~ `</name>
213     <title>` ~ result.name ~ `</title>
214     <test-cases>
215     </test-cases>
216     <attachments>
217       <attachment title="title" source="` ~ uuid ~ `/title.0.some_text.txt" type="plain/text" />
218     </attachments>
219     <labels>
220         <label name="framework" value="Trial"/>
221         <label name="language" value="D"/>
222     </labels>
223 </ns2:test-suite>`);
224 }
225 
226 struct AllureTestXml {
227   ///
228   TestResult result;
229 
230   ///
231   string uuid;
232 
233   private {
234     immutable string destination;
235   }
236 
237   this(const string destination, TestResult result, string uuid) {
238     this.destination = destination;
239     this.result = result;
240     this.uuid = uuid;
241   }
242 
243   /// Converts a test result to allure status
244   string allureStatus() {
245     switch(result.status) {
246       case TestResult.Status.created:
247         return "canceled";
248 
249       case TestResult.Status.failure:
250         return "failed";
251 
252       case TestResult.Status.skip:
253         return "canceled";
254 
255       case TestResult.Status.success:
256         return "passed";
257 
258       default:
259         return "unknown";
260     }
261   }
262 
263   /// Return the string representation of the test
264   string toString() {
265     auto epoch = SysTime.fromUnixTime(0);
266     auto start = (result.begin - epoch).total!"msecs";
267     auto stop = (result.end - epoch).total!"msecs";
268 
269     string xml = `        <test-case start="` ~ start.to!string ~ `" stop="` ~ stop.to!string ~ `" status="` ~ allureStatus ~ `">` ~ "\n";
270     xml ~= `            <name>` ~ result.name.escape ~ `</name>` ~ "\n";
271 
272     if(result.labels.length > 0) {
273       xml ~= "            <labels>\n";
274 
275       foreach(label; result.labels) {
276         xml ~= "              <label name=\"" ~ label.name ~ "\" value=\"" ~ label.value ~ "\"/>\n";
277       }
278 
279       xml ~= "            </labels>\n";
280     }
281 
282     if(result.throwable !is null) {
283       xml ~= `            <failure>
284                 <message>` ~ result.throwable.msg.escape ~ `</message>
285                 <stack-trace>` ~ result.throwable.to!string.escape ~ `</stack-trace>
286             </failure>` ~ "\n";
287     }
288 
289     if(result.steps.length > 0) {
290       xml ~= "            <steps>\n";
291       xml ~= result.steps.map!(a => AllureStepXml(destination, a, 14, uuid)).map!(a => a.toString).array.join('\n') ~ "\n";
292       xml ~= "            </steps>\n";
293     }
294 
295     if(result.attachments.length > 0) {
296       xml ~= "            <attachments>\n";
297       xml ~= result.attachments.map!(a => AllureAttachmentXml(destination, a, 14, uuid)).map!(a => a.toString).array.join('\n') ~ "\n";
298       xml ~= "            </attachments>\n";
299     }
300 
301     xml ~= `        </test-case>`;
302 
303     return xml;
304   }
305 }
306 
307 @("AllureTestXml should transform a success test")
308 unittest
309 {
310   auto epoch = SysTime.fromUnixTime(0);
311 
312   TestResult result = new TestResult("Test");
313   result.begin = Clock.currTime;
314   result.end = Clock.currTime;
315   result.status = TestResult.Status.success;
316 
317   auto allure = AllureTestXml("allure", result, "");
318 
319   allure.toString.should.equal(
320   `        <test-case start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
321             <name>Test</name>
322         </test-case>`);
323 }
324 
325 @("AllureTestXml should transform a failing test")
326 unittest
327 {
328   import trial.step;
329 
330   Step("prepare the test data");
331   auto epoch = SysTime.fromUnixTime(0);
332   TestResult result = new TestResult("Test");
333   result.begin = Clock.currTime;
334   result.end = Clock.currTime;
335   result.status = TestResult.Status.failure;
336   result.throwable = new Exception("message");
337 
338   Step("create the report listener");
339   auto allure = AllureTestXml("allure", result, "");
340 
341   Step("perform checks");
342   allure.toString.should.equal(
343  `        <test-case start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="failed">
344             <name>Test</name>
345             <failure>
346                 <message>message</message>
347                 <stack-trace>object.Exception@lifecycle/trial/reporters/allure.d(` ~ result.throwable.line.to!string ~ `): message</stack-trace>
348             </failure>
349         </test-case>`);
350 }
351 
352 /// AllureTestXml should transform a test with steps
353 unittest
354 {
355   auto epoch = SysTime.fromUnixTime(0);
356 
357   TestResult result = new TestResult("Test");
358   result.begin = Clock.currTime;
359   result.end = Clock.currTime;
360   result.status = TestResult.Status.success;
361 
362   StepResult step = new StepResult();
363   step.name = "some step";
364   step.begin = result.begin;
365   step.end = result.end;
366 
367   result.steps = [step, step];
368 
369   auto allure = AllureTestXml("allure", result, "");
370 
371   allure.toString.should.equal(
372   `        <test-case start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
373             <name>Test</name>
374             <steps>
375                 <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
376                   <name>some step</name>
377                 </step>
378                 <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
379                   <name>some step</name>
380                 </step>
381             </steps>
382         </test-case>`);
383 }
384 
385 /// AllureTestXml should transform a test with labels
386 unittest
387 {
388   auto epoch = SysTime.fromUnixTime(0);
389 
390   TestResult result = new TestResult("Test");
391   result.begin = Clock.currTime;
392   result.end = Clock.currTime;
393   result.status = TestResult.Status.success;
394   result.labels ~= Label("status_details", "flaky");
395 
396   auto allure = AllureTestXml("allure", result, "");
397 
398   allure.toString.should.equal(
399  `        <test-case start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
400             <name>Test</name>
401             <labels>
402               <label name="status_details" value="flaky"/>
403             </labels>
404         </test-case>`);
405 }
406 
407 /// AllureTestXml should add the attachments
408 unittest
409 {
410   string resource = buildPath(getcwd(), "some_text.txt");
411   std.file.write(resource, "");
412 
413   auto uuid = randomUUID.toString;
414 
415   scope(exit) {
416     if(exists(resource)) {
417       remove(resource);
418     }
419 
420     remove("allure/" ~ uuid ~ "/title.0.some_text.txt");
421   }
422 
423   auto epoch = SysTime.fromUnixTime(0);
424 
425   TestResult result = new TestResult("Test");
426   result.begin = Clock.currTime;
427   result.end = Clock.currTime;
428   result.status = TestResult.Status.success;
429   result.attachments = [ Attachment("title", resource, "plain/text") ];
430 
431   auto allure = AllureTestXml("allure", result, uuid);
432 
433   allure.toString.should.equal(
434  `        <test-case start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
435             <name>Test</name>
436             <attachments>
437               <attachment title="title" source="` ~ uuid ~ `/title.0.some_text.txt" type="plain/text" />
438             </attachments>
439         </test-case>`);
440 }
441 
442 struct AllureStepXml {
443   private {
444     StepResult step;
445     size_t indent;
446     string uuid;
447 
448     immutable string destination;
449   }
450 
451   this(const string destination, StepResult step, size_t indent, string uuid) {
452     this.step = step;
453     this.indent = indent;
454     this.uuid = uuid;
455     this.destination = destination;
456   }
457 
458   /// Return the string representation of the step
459   string toString() {
460     auto epoch = SysTime.fromUnixTime(0);
461     const spaces = "  " ~ (" ".repeat(indent).array.join());
462     string result = spaces ~ `<step start="` ~ (step.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (step.end - epoch).total!"msecs".to!string ~ `" status="passed">` ~ "\n" ~
463     spaces ~ `  <name>` ~ step.name.escape ~ `</name>` ~ "\n";
464 
465     if(step.steps.length > 0) {
466       result ~= spaces ~ "  <steps>\n";
467       result ~= step.steps.map!(a => AllureStepXml(destination, a, indent + 6, uuid)).map!(a => a.to!string).array.join('\n') ~ "\n";
468       result ~= spaces ~ "  </steps>\n";
469     }
470 
471     if(step.attachments.length > 0) {
472       result ~= spaces ~ "  <attachments>\n";
473       result ~= step.attachments.map!(a => AllureAttachmentXml(destination, a, indent + 6, uuid)).map!(a => a.to!string).array.join('\n') ~ "\n";
474       result ~= spaces ~ "  </attachments>\n";
475     }
476 
477     result ~= spaces ~ `</step>`;
478 
479     return result;
480   }
481 }
482 
483 /// AllureStepXml should transform a step
484 unittest
485 {
486   auto epoch = SysTime.fromUnixTime(0);
487   StepResult result = new StepResult();
488   result.name = "step";
489   result.begin = Clock.currTime;
490   result.end = Clock.currTime;
491 
492   auto allure = AllureStepXml("allure", result, 0, "");
493 
494   allure.toString.should.equal(
495   `  <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
496     <name>step</name>
497   </step>`);
498 }
499 
500 /// AllureStepXml should transform nested steps
501 unittest
502 {
503   auto epoch = SysTime.fromUnixTime(0);
504   StepResult result = new StepResult();
505   result.name = "step";
506   result.begin = Clock.currTime;
507   result.end = Clock.currTime;
508 
509   StepResult step = new StepResult();
510   step.name = "some step";
511   step.begin = result.begin;
512   step.end = result.end;
513 
514   result.steps = [ step, step ];
515 
516   auto allure = AllureStepXml("allure", result, 0, "");
517 
518   allure.toString.should.equal(
519   `  <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
520     <name>step</name>
521     <steps>
522         <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
523           <name>some step</name>
524         </step>
525         <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
526           <name>some step</name>
527         </step>
528     </steps>
529   </step>`);
530 }
531 
532 /// AllureStepXml should add the attachments
533 unittest
534 {
535   string resource = buildPath(getcwd(), "some_image.png");
536   scope(exit) {
537     resource.remove();
538   }
539   std.file.write(resource, "");
540 
541   auto uuid = randomUUID.toString;
542 
543   scope(exit) {
544     rmdirRecurse("allure");
545   }
546 
547 
548   auto epoch = SysTime.fromUnixTime(0);
549   StepResult result = new StepResult();
550   result.name = "step";
551   result.begin = Clock.currTime;
552   result.end = Clock.currTime;
553 
554   result.attachments = [ Attachment("name", resource, "image/png") ];
555 
556   auto allure = AllureStepXml("allure", result, 0, uuid);
557 
558   allure.toString.should.equal(
559   `  <step start="` ~ (result.begin - epoch).total!"msecs".to!string ~ `" stop="` ~ (result.end - epoch).total!"msecs".to!string ~ `" status="passed">
560     <name>step</name>
561     <attachments>
562       <attachment title="name" source="` ~ uuid ~ `/name.0.some_image.png" type="image/png" />
563     </attachments>
564   </step>`);
565 }
566 
567 /// Allure representation of an atachment.
568 /// It will copy the file to the allure folder with an unique name
569 struct AllureAttachmentXml {
570 
571   private {
572     const {
573       Attachment attachment;
574       size_t indent;
575     }
576 
577     string allureFile;
578   }
579 
580   @disable this();
581 
582   /// Init the struct and copy the atachment to the allure folder
583   this(const string destination, Attachment attachment, size_t indent, string uuid) {
584     this.indent = indent;
585 
586     if(!exists(buildPath(destination, uuid))) {
587       buildPath(destination, uuid).mkdirRecurse;
588     }
589 
590     ulong index;
591 
592     do {
593       allureFile = buildPath(uuid, attachment.name ~ "." ~ index.to!string ~ "." ~ baseName(attachment.file));
594       index++;
595     } while(buildPath(destination, allureFile).exists);
596 
597     if(attachment.file.exists) {
598       std.file.copy(attachment.file, buildPath(destination, allureFile));
599     }
600 
601     this.attachment = Attachment(attachment.name, buildPath(destination, allureFile), attachment.mime);
602   }
603 
604   /// convert the attachment to string
605   string toString() {
606     return (" ".repeat(indent).array.join()) ~ "<attachment title=\"" ~ attachment.name ~
607       "\" source=\"" ~ allureFile ~
608       "\" type=\"" ~ attachment.mime ~ "\" />";
609   }
610 }
611 
612 /// Allure attachments should be copied to a folder containing the suite name
613 unittest {
614   string resource = buildPath(getcwd(), "some_image.png");
615   std.file.write(resource, "");
616 
617   auto uuid = randomUUID.toString;
618   auto expectedPath = buildPath(getcwd(), "allure",  uuid, "name.0.some_image.png");
619 
620   scope(exit) {
621     rmdirRecurse("allure");
622   }
623 
624   auto a = AllureAttachmentXml("allure", Attachment("name", resource, ""), 0, uuid);
625 
626   expectedPath.exists.should.equal(true);
627 }
628 
629 /// Allure attachments should avoid name collisions
630 unittest {
631   string resource = buildPath(getcwd(), "some_image.png");
632   std.file.write(resource, "");
633 
634   auto uuid = randomUUID.toString;
635 
636   buildPath(getcwd(), "allure",  uuid).mkdirRecurse;
637   auto expectedPath = buildPath(getcwd(), "allure", uuid, "name.1.some_image.png");
638   auto existingPath = buildPath(getcwd(), "allure", uuid, "name.0.some_image.png");
639   std.file.write(existingPath, "");
640 
641   scope(exit) {
642     rmdirRecurse("allure");
643   }
644 
645   auto a = AllureAttachmentXml("allure", Attachment("name", resource, ""), 0, uuid);
646 
647   expectedPath.exists.should.equal(true);
648 }