A tool that tries to build every Dub package against a wide range of DMD versions.

html.d 20KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
  1. /++
  2. Produces HTML reports for our results.
  3. +/
  4. module dubautotester.html;
  5. import dubautotester.core;
  6. import std.algorithm;
  7. import std.array;
  8. import std.conv;
  9. import std.datetime.systime;
  10. import std.experimental.logger;
  11. import std.file;
  12. import std.path;
  13. import std.range;
  14. import std.stdio : File;
  15. void append(K, V)(ref V[][K] mm, K key, V value)
  16. {
  17. if (auto p = key in mm) *p ~= value;
  18. else mm[key] = [value];
  19. }
  20. V get(K, V)(V[K] aa, K key)
  21. {
  22. if (auto p = key in aa) return *p;
  23. return V.init;
  24. }
  25. enum css = `
  26. td.success {
  27. background-color: #B8F3C5;
  28. }
  29. td.failure {
  30. background-color: #F3B8BF;
  31. }
  32. .tableheader {
  33. font-weight: bold;
  34. text-decoration: underline;
  35. }
  36. .same {
  37. background-color: #C7DBF9;
  38. }
  39. .fixed {
  40. background-color: #C7F9CC;
  41. }
  42. .broken {
  43. background-color: #F9C7C8;
  44. }
  45. table {
  46. border-collapse: collapse;
  47. }
  48. td {
  49. padding-left: 0.5em;
  50. padding-right: 0.5em;
  51. border: 1px dotted gray;
  52. text-align: center;
  53. }
  54. tr:nth-child(2n) {
  55. background: #f0f0f0;
  56. }
  57. `;
  58. enum standardDisclaimer = [
  59. `This is a best-effort attempt to test all dub packages against all dmd versions
  60. (at least since DMD 2.070.1; will try to get older versions.) It simply checks
  61. out each version and runs dub build and dub test.`,
  62. `The build system is Linux/amd64. Some projects may fail here that succeed on
  63. other OSes and platforms, and some projects may succeed here that fail
  64. elsewhere. Similarly, DMD is the only compiler tested at the moment. This report
  65. is meant to be informative, but not definitive.`,
  66. `This is not a replacement for a proper CI system. It only runs dub test, which
  67. is not sufficient for many packages. For instance, while vibe-d has many
  68. unittests, these unittests are not sufficient to determine if the code actually
  69. works. On the other hand, it's a good indication of whether certain projects
  70. that are more data-driven than interaction-driven, such as quantities, work.`,
  71. `A small number of packages are blacklisted, including csprng and qte5. The
  72. former takes far too long to test (multiple hours at a minimum) and the latter
  73. creates X11 windows and doesn't automatically close them.`,
  74. `As of 2018-09-09, the data about deprecated code is incomplete. Compiler
  75. version 2.082.0 has no deprecation data, while 2.078.1 has partial data.`,
  76. `Questions? Comments? Issues? Contact neia@ikeran.org`,
  77. ];
  78. void buildEfficient(string path, string title, SysTime lastUpdate, void delegate(ref File) main)
  79. {
  80. static SysTime exeDate = SysTime.init;
  81. import std.file : thisExePath, timeLastModified, exists;
  82. if (exeDate == SysTime.init)
  83. {
  84. exeDate = timeLastModified(thisExePath);
  85. }
  86. if (exeDate > lastUpdate)
  87. {
  88. lastUpdate = exeDate;
  89. }
  90. if (exists(path))
  91. {
  92. if (timeLastModified(path) >= lastUpdate)
  93. {
  94. tracef("%s last modified is %s; already up to date", path, timeLastModified(path));
  95. return;
  96. }
  97. }
  98. auto rel = (path.endsWith("index.html")) ? "" : "../";
  99. auto f = File(path, "w");
  100. f.writef(`<!DOCTYPE html>
  101. <html>
  102. <head>
  103. <meta charset="utf-8">
  104. <script type="text/javascript" src="%1$sautotester.js"></script>
  105. <link rel="stylesheet" href="%1$sautotester.css">
  106. <title>%2%s</title>
  107. </head>
  108. <body>
  109. <a href="%1$sindex.html">Main page</a>
  110. <h1>%2$s</h1>`, rel, title);
  111. main(f);
  112. f.write(`
  113. </body>
  114. </html> `);
  115. f.flush;
  116. f.close;
  117. }
  118. void buildIndex(string path, Config config)
  119. {
  120. buildEfficient(path, "dub autotester: main report", SysTime.max, (ref File f)
  121. {
  122. f.write(`<table>
  123. <thead>
  124. <tr>
  125. <th>DMD version</th>
  126. <th>Packages attempted</th>
  127. <th>Checked out</th>
  128. <th>Builds</th>
  129. <th>Passes tests</th>
  130. <th>Passes deprecation checks</th>
  131. </tr>
  132. </thead>
  133. <tbody>`);
  134. auto summaries = config.getCompilerSummaries;
  135. foreach (summary; summaries)
  136. {
  137. auto deprecationPasses = summary.packageCount - summary.deprecationCount;
  138. f.writef(`
  139. <tr>
  140. <td><a href="compiler/dmd%s.html">%1$s</a></td>
  141. <td>%2$s</td>
  142. <td>%3$s (%4$#.1f%%)</td>
  143. <td>%5$s (%6$#.1f%%)</td>
  144. <td>%7$s (%8$#.1f%%)</td>
  145. <td>%9$s (%10$#.1f%%)</td>
  146. </tr>`,
  147. summary.release,
  148. summary.packageCount,
  149. summary.checkoutCount,
  150. (100.0 * summary.checkoutCount / summary.packageCount),
  151. summary.buildCount,
  152. (100.0 * summary.buildCount / summary.packageCount),
  153. summary.testCount,
  154. (100.0 * summary.testCount / summary.packageCount),
  155. deprecationPasses,
  156. (100.0 * deprecationPasses / summary.packageCount)
  157. );
  158. }
  159. f.write(`
  160. </tbody>
  161. </table>`);
  162. });
  163. }
  164. void buildCompilerPage(Config config, string path, string release, string[] diffVersions)
  165. {
  166. auto maxAge = config.mostRecentCompilerBuild(release);
  167. buildEfficient(path, release ~ " report", maxAge, (ref File f)
  168. {
  169. if (diffVersions.length > 0)
  170. {
  171. f.write(`
  172. <div>
  173. Diff with:`);
  174. foreach (v; diffVersions)
  175. {
  176. string a, b;
  177. if (v < release)
  178. {
  179. a = v;
  180. b = release;
  181. }
  182. else
  183. {
  184. a = release;
  185. b = v;
  186. }
  187. f.writef(` <a href="../diffs/%s_to_%s.html">%s</a>`, a, b, v);
  188. }
  189. f.write(`
  190. </div>`);
  191. }
  192. auto builds = config.findBuildsByCompiler(release).array;
  193. buildPackageReportStuff(f, builds, Type.compilers);
  194. });
  195. }
  196. void buildPackagePage(Config config, string packageName, string path, Build[] builds)
  197. {
  198. auto age = config.mostRecentPackageBuild(packageName);
  199. buildEfficient(path, builds[0].packageName ~ " report", age, (ref File f)
  200. {
  201. buildPackageReportStuff(f, builds, Type.compilers);
  202. });
  203. }
  204. void buildPackageReportStuff(ref File f, Build[] builds, Type linkTo)
  205. {
  206. builds.sort!(buildLess);
  207. {
  208. f.write(`<table>
  209. <thead>
  210. <tr>
  211. <th>Package</th>
  212. <th>Version</th>
  213. <th>DMD version</th>
  214. <th>Checks out?</th>
  215. <th>Builds?</th>
  216. <th>Passes tests?</th>
  217. <th>Passes deprecation checks?</th>
  218. <th>Build log</th>
  219. </tr>
  220. </thead>
  221. <tbody>`);
  222. foreach (i, build; builds)
  223. {
  224. if (linkTo & Type.packages)
  225. {
  226. f.writef(`
  227. <tr>
  228. <td><a href="../package/%1$s.html">%1$s</a></td>`, build.packageName);
  229. }
  230. else
  231. {
  232. f.writef(`
  233. <tr>
  234. <td>%1$s</td>`, build.packageName);
  235. }
  236. f.writef(`
  237. <td>%s</td>`, build.revisionId);
  238. if (linkTo & Type.compilers)
  239. {
  240. f.writef(`
  241. <td><a href="../compiler/dmd%1$s.html">%1$s</a></td>`, build.compilerRelease);
  242. }
  243. else
  244. {
  245. f.writef(`
  246. <td>%1$s</td>`, build.compilerRelease);
  247. }
  248. foreach (b; [build.canCheckOut, build.canBuild, build.canTest, !build.hasDeprecations])
  249. {
  250. f.writef(`
  251. <td class="%1$s">%1$s</td>`, b ? "success" : "failure");
  252. }
  253. if (build.stderr)
  254. {
  255. f.writef(`
  256. <td>
  257. <a href="#build%1$s" id="build%1$s" onclick="showBuildOutput(build%1$s)"
  258. data-stderr="%2$s">Build log</a>
  259. </td>`,
  260. i, build.stderr.htmlEntitiesEncode);
  261. }
  262. else
  263. {
  264. f.write(`<td></td>`);
  265. }
  266. f.writef(`
  267. </tr>`);
  268. }
  269. f.write(`
  270. </tbody>
  271. </table>`);
  272. }
  273. }
  274. bool semverLess(string a, string b)
  275. {
  276. auto ap = a.splitter('.');
  277. auto bp = a.splitter('.');
  278. while (!ap.empty && !bp.empty)
  279. {
  280. auto ma = ap.front.splitter('-').front;
  281. auto mb = bp.front.splitter('-').front;
  282. if (ma == mb)
  283. {
  284. if (ap.front.canFind('-'))
  285. {
  286. return true;
  287. }
  288. if (bp.front.canFind('-'))
  289. {
  290. return false;
  291. }
  292. ap.popFront;
  293. bp.popFront;
  294. continue;
  295. }
  296. return ma.to!uint < mb.to!uint;
  297. }
  298. return false;
  299. }
  300. bool buildLess(const ref Build a, const ref Build b)
  301. {
  302. if (a.packageName < b.packageName) return true;
  303. if (a.packageName > b.packageName) return false;
  304. if (semverLess(a.revisionId, b.revisionId)) return false;
  305. if (semverLess(b.revisionId, a.revisionId)) return true;
  306. return a.compilerRelease < b.compilerRelease;
  307. }
  308. void buildCompilerReports(Config config)
  309. {
  310. import std.datetime.stopwatch : StopWatch;
  311. ulong count = 0;
  312. StopWatch sw;
  313. sw.start;
  314. std.file.write(config.reportsDir.buildPath("autotester.js"), js);
  315. std.file.write(config.reportsDir.buildPath("autotester.css"), css);
  316. string[][string] haveDiffs;
  317. tracef("building compiler diffs!");
  318. auto releases = config.findCompilerReleases;
  319. tracef("have %s releases to go through", releases.length);
  320. auto diffDir = buildPath(config.reportsDir, "diffs");
  321. mkdirRecurse(diffDir);
  322. foreach (i, a; releases)
  323. {
  324. foreach (j, b; releases)
  325. {
  326. if (j >= i) break;
  327. import std.format : format;
  328. tracef("diff %s vs %s", a, b);
  329. buildDiff(
  330. config,
  331. buildPath(diffDir, "%s_to_%s.html".format(a, b)),
  332. a,
  333. b);
  334. haveDiffs.append(a, b);
  335. haveDiffs.append(b, a);
  336. count++;
  337. }
  338. }
  339. tracef("building compiler reports!");
  340. auto dir = config.reportsDir.buildPath("compiler");
  341. dir.mkdirRecurse;
  342. tracef("individual reports will go in %s", dir);
  343. auto last = "";
  344. foreach (release; releases)
  345. {
  346. count++;
  347. tracef("working with release %s", release);
  348. buildCompilerPage(
  349. config,
  350. dir.buildPath("dmd" ~ release ~ ".html"),
  351. release,
  352. haveDiffs.get(release));
  353. tracef("write report for %s", release);
  354. }
  355. /* Have compilers, have index, need packages. */
  356. auto pkgs = config.findPackageNames;
  357. auto pkgDir = config.reportsDir.buildPath("package");
  358. mkdirRecurse(pkgDir);
  359. foreach (name; pkgs)
  360. {
  361. count++;
  362. auto builds = config.findBuildsByPackage(name).array;
  363. writeBadge(pkgDir, name, builds);
  364. buildPackagePage(config, name, pkgDir.buildPath(name ~ ".html"), builds);
  365. tracef("write report for %s", name);
  366. }
  367. buildIndex(buildPath(config.reportsDir, "index.html"), config);
  368. sw.stop;
  369. auto duration = sw.peek;
  370. infof("wrote %s compiler reports to %s in %s!", count, config.reportsDir, duration);
  371. }
  372. bool sameMajorVersion(string a, string b)
  373. {
  374. import std.string : lastIndexOf;
  375. a = a[0..a.lastIndexOf('.')];
  376. b = b[0..b.lastIndexOf('.')];
  377. return a == b;
  378. }
  379. bool sameResult(Build olderBuild, Build newerBuild)
  380. {
  381. return olderBuild.canBuild == newerBuild.canBuild &&
  382. olderBuild.canTest == newerBuild.canTest &&
  383. olderBuild.hasDeprecations == newerBuild.hasDeprecations;
  384. }
  385. void buildDiff(Config config, string path, string older, string newer)
  386. {
  387. import std.exception : enforce;
  388. enforce(newer > older);
  389. string delta(bool oldWorks, bool newWorks)
  390. {
  391. if (oldWorks == newWorks) return "same";
  392. if (!newWorks) return "broken";
  393. return "fixed";
  394. }
  395. bool newBreakage(Build olderBuild, Build newerBuild)
  396. {
  397. enforce(olderBuild.compilerRelease < newerBuild.compilerRelease);
  398. enforce(olderBuild.compilerRelease == older);
  399. enforce(newerBuild.compilerRelease == newer);
  400. return (!olderBuild.canBuild && newerBuild.canBuild) ||
  401. (!olderBuild.canTest && newerBuild.canTest) ||
  402. (!olderBuild.hasDeprecations && newerBuild.hasDeprecations);
  403. }
  404. auto oldBuilds = config.findBuildsByCompiler(older).array;
  405. foreach (build; oldBuilds)
  406. {
  407. enforce(build.compilerRelease == older);
  408. }
  409. auto newBuilds = config.findBuildsByCompiler(newer).array;
  410. foreach (build; newBuilds)
  411. {
  412. enforce(build.compilerRelease == newer);
  413. }
  414. auto maxAgeA = config.mostRecentCompilerBuild(older);
  415. auto maxAgeB = config.mostRecentCompilerBuild(newer);
  416. auto maxAge = maxAgeA > maxAgeB ? maxAgeA : maxAgeB;
  417. import std.format : format;
  418. buildEfficient(path, "Diff: %s vs %s".format(older, newer), maxAge, (ref File f)
  419. {
  420. f.write(`
  421. <table>
  422. <thead>
  423. <tr>
  424. <th>Package name</th>
  425. <th>Revision</th>
  426. <th>Builds?</th>
  427. <th>Tests?</th>
  428. <th>Deprecations?</th>
  429. <th>Build log</th>
  430. </tr>
  431. </thead>
  432. <tbody>`);
  433. ulong count = 0;
  434. ulong changes = 0;
  435. // Same ordering, so we can iterate once to find diffs.
  436. while (!oldBuilds.empty && !newBuilds.empty)
  437. {
  438. // Ordered by packagename ascending, revision id ascending
  439. auto olderBuild = oldBuilds.front;
  440. auto newerBuild = newBuilds.front;
  441. enforce(olderBuild.compilerRelease == older);
  442. enforce(newerBuild.compilerRelease == newer);
  443. if (olderBuild.packageName < newerBuild.packageName)
  444. {
  445. oldBuilds.popFront;
  446. continue;
  447. }
  448. else if (olderBuild.packageName > newerBuild.packageName)
  449. {
  450. newBuilds.popFront;
  451. continue;
  452. }
  453. if (olderBuild.revisionId < newerBuild.revisionId)
  454. {
  455. oldBuilds.popFront;
  456. continue;
  457. }
  458. else if (olderBuild.revisionId > newerBuild.revisionId)
  459. {
  460. newBuilds.popFront;
  461. continue;
  462. }
  463. count++;
  464. oldBuilds.popFront;
  465. newBuilds.popFront;
  466. // It's not interesting if they can't check out.
  467. if (!olderBuild.canCheckOut || !newerBuild.canCheckOut) continue;
  468. if (sameResult(olderBuild, newerBuild)) continue;
  469. enforce(olderBuild.compilerRelease == older);
  470. enforce(newerBuild.compilerRelease == newer);
  471. changes++;
  472. f.writef(`
  473. <tr>
  474. <td>%1$s</td>
  475. <td>%2$s</td>
  476. <td class="%3$s">%3$s</td>
  477. <td class="%4$s">%4$s</td>
  478. <td class="%5$s">%5$s</td>`,
  479. olderBuild.packageName,
  480. olderBuild.revisionId,
  481. delta(olderBuild.canBuild, newerBuild.canBuild),
  482. delta(olderBuild.canTest, newerBuild.canTest),
  483. delta(!olderBuild.hasDeprecations, !newerBuild.hasDeprecations));
  484. // Over-supply build logs
  485. if (newerBuild.stderr.length > 0)
  486. {
  487. f.writef(`
  488. <td>
  489. <a href="#build%1$s" id="build%1$s" onclick="showBuildOutput(build%1$s)"
  490. data-stderr="%2$s">Build log</a>
  491. </td>
  492. </tr>`,
  493. count, newerBuild.stderr.htmlEntitiesEncode);
  494. }
  495. else
  496. {
  497. f.writef(`
  498. <td>&mdash;</td>
  499. </tr>`);
  500. }
  501. }
  502. f.writef(`
  503. </table>
  504. <div>%s packages differed over %s mutually tested packages</div>`, changes, count);
  505. });
  506. }
  507. void writeBadge(string dir, string pkgName, Build[] builds)
  508. {
  509. import std.typecons : tuple;
  510. import std.stdio : File;
  511. // Order by compilerRelease descending
  512. builds.sort!((x, y) => x.compilerRelease > y.compilerRelease);
  513. auto successes = builds
  514. .chunkBy!(x => x.compilerRelease)
  515. .map!(xs => tuple(xs[0], xs[1].any!(x => x.canBuild && x.canTest)))
  516. .array;
  517. string start, end;
  518. foreach (i, v; successes)
  519. {
  520. if (v[1])
  521. {
  522. if (start == null) start = v[0];
  523. }
  524. else if (start != null)
  525. {
  526. end = successes[i-1][0];
  527. break;
  528. }
  529. }
  530. string svg;
  531. if (start == null)
  532. {
  533. start = "😞 none";
  534. end = "😞 none";
  535. }
  536. auto f = File(chainPath(dir, pkgName ~ ".svg"), "w");
  537. f.writefln(basicSvg, start, end);
  538. f.flush;
  539. f.close;
  540. }
  541. enum basicSvg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 40">
  542. <rect x="0" y="0" width="100" height="40"
  543. style="stroke: #000000; fill: #efefef" />
  544. <text x="1" y="36"
  545. style="fill: #991c0c; stroke: #000000; stroke-width: 1;
  546. font-size: 40px; font-family: sans-serif">D</text>
  547. <text x="36" y="36"
  548. style="color: #000000; font-size: 15px;
  549. font-family: sans-serif">%s</text>
  550. <text x="36" y="16"
  551. style="color: #000000; font-size: 15px;
  552. font-family: sans-serif">%s</text>
  553. <line x1="52" y1="20" x2="82" y2="20" stroke="#000000" />
  554. </svg>`;
  555. enum Type
  556. {
  557. nothing = 0,
  558. packages = 1,
  559. compilers = 2,
  560. both = 3,
  561. }
  562. string utfSafe(string s)
  563. {
  564. import std.array : Appender;
  565. import std.utf;
  566. Appender!dstring ap;
  567. size_t i;
  568. while (i < s.length)
  569. {
  570. ap ~= decode!(Yes.useReplacementDchar)(s, i);
  571. }
  572. return ap.data.to!string;
  573. }
  574. unittest
  575. {
  576. auto s = utfSafe("hello\xFF\xFAworld");
  577. assert(s == "hello\uFFFDworld");
  578. }
  579. enum js = `
  580. function showBuildOutput(elem) {
  581. var popup = document.getElementById("popup");
  582. if (popup == null) {
  583. popup = document.createElement("div");
  584. popup.id = "popup";
  585. popup.style.position = "fixed";
  586. popup.style.top = "4em";
  587. popup.style.left = "4em";
  588. popup.style.right = "4em";
  589. popup.style.bottom = "4em";
  590. popup.style.border = "4px solid red;"
  591. popup.style.padding = "2em"
  592. popup.style.backgroundColor = "#ddd";
  593. popup.style.overflow = "scroll";
  594. let pre = document.createElement("pre");
  595. pre.id = "stderr";
  596. popup.appendChild(pre);
  597. var button = document.createElement("button");
  598. button.innerText = "Close";
  599. button.addEventListener("click", function() { popup.style.display = "none"; });
  600. popup.appendChild(button);
  601. document.body.appendChild(popup);
  602. }
  603. popup.style.display = "block";
  604. var stderrBlock = document.getElementById("stderr");
  605. stderrBlock.innerText = elem.dataset.stderr;
  606. }
  607. document.addEventListener("keypress", (evt) => {
  608. if (evt.key != "Escape") return;
  609. let popup = document.getElementById("popup");
  610. if (popup == null) return;
  611. popup.style.display = "none";
  612. });
  613. `;
  614. string htmlEntitiesEncode(string data)
  615. {
  616. // Inspired largely by arsd.dom. Strips invalid UTF characters.
  617. foreach (char c; data)
  618. {
  619. if (c > 127 || c == '<' || c == '>' || c == '&' || c == '"')
  620. {
  621. // We need to encode, go for it.
  622. goto doEncode;
  623. }
  624. }
  625. // Didn't find anything to encode? Just return the input.
  626. return data;
  627. doEncode:
  628. import std.utf : decode;
  629. Appender!string s;
  630. s.reserve(data.length + data.length / 16 + 32);
  631. size_t n;
  632. while (n < data.length)
  633. {
  634. auto c = decode!(Yes.useReplacementDchar)(data, n);
  635. if (c > 127)
  636. {
  637. s.put("&#");
  638. s.put((cast(uint)c).to!string);
  639. s.put(";");
  640. }
  641. else if (c == '&')
  642. {
  643. s.put("&amp;");
  644. }
  645. else if (c == '<')
  646. {
  647. s.put("&lt;");
  648. }
  649. else if (c == '>')
  650. {
  651. s.put("&gt;");
  652. }
  653. else if (c == '"')
  654. {
  655. s.put("&quot;");
  656. }
  657. else
  658. {
  659. s.put(c);
  660. }
  661. }
  662. return s.data;
  663. }