A tool that tries to build every Dub package against a wide range of DMD versions.

packagefinder.d 3.4KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125
  1. module backporter.packagefinder;
  2. import std.algorithm, std.array;
  3. import backporter.core;
  4. import backporter.git;
  5. import std.stdio;
  6. import std.file;
  7. import std.json;
  8. import std.path;
  9. import std.experimental.logger;
  10. JSONValue getPackageJson(string name)
  11. {
  12. try
  13. {
  14. import std.net.curl : get;
  15. infof("grabbing package %s", name);
  16. auto js = get("https://code.dlang.org/api/packages/" ~ name ~ "/info").parseJSON;
  17. return js;
  18. }
  19. catch (Exception e)
  20. {
  21. return JSONValue(null);
  22. }
  23. }
  24. string[] packageNames()
  25. {
  26. import std.net.curl : get;
  27. infof("grabbing package list");
  28. auto js = get("https://code.dlang.org/packages/index.json").parseJSON.array;
  29. auto names = new string[js.length];
  30. foreach (i, n; js)
  31. {
  32. names[i] = n.str;
  33. }
  34. names = names.sort.uniq.array;
  35. infof("found %s packages", names.length);
  36. return names;
  37. }
  38. auto parsePackages(JSONValue value, Config config)
  39. {
  40. PackageRevision[] revisions;
  41. auto name = value["name"].str;
  42. auto repo = value["repository"];
  43. string gitUrl;
  44. switch (repo["kind"].str)
  45. {
  46. case "github":
  47. gitUrl = "https://github.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str;
  48. break;
  49. case "gitlab":
  50. gitUrl = "https://gitlab.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str ~ ".git";
  51. break;
  52. case "bitbucket":
  53. gitUrl = "https://bitbucket.org/" ~ repo["owner"].str ~ "/" ~ repo["project"].str ~ ".git";
  54. break;
  55. default:
  56. errorf("failed to parse repo info for project %s", name);
  57. break;
  58. }
  59. auto v = value["versions"].array;
  60. v.sort!((a, b) => (a["date"].str > b["date"].str));
  61. foreach (revision; value["versions"].array)
  62. {
  63. if (!("version" in revision))
  64. {
  65. warningf("package %s contains a revision without a version name", name);
  66. continue;
  67. }
  68. if (!("commitID" in revision))
  69. {
  70. warningf("package %s contains a revision without a commit id", name);
  71. continue;
  72. }
  73. auto id = revision["version"].str;
  74. if (id.canFind('-')) continue;
  75. if (id.canFind('~')) continue;
  76. PackageRevision rev;
  77. rev.commitId = revision["commitID"].str;
  78. rev.revisionId = id;
  79. rev.packageName = name;
  80. rev.gitUrl = gitUrl;
  81. revisions ~= rev;
  82. }
  83. if (revisions.length <= config.revisionCount)
  84. {
  85. return revisions;
  86. }
  87. return revisions[0..config.revisionCount];
  88. }
  89. PackageRevision[] grabAllFresh(Config config)
  90. {
  91. auto names = packageNames;
  92. auto packageJsons = names
  93. .map!getPackageJson
  94. .filter!(x => x.type == JSON_TYPE.OBJECT)
  95. .array;
  96. infof("downloading packages fresh");
  97. JSONValue v;
  98. v["packages"] = packageJsons;
  99. auto f = File("packagelist.json", "w");
  100. f.write(v.toString);
  101. f.close;
  102. return packageJsons.map!(x => x.parsePackages(config)).joiner.array;
  103. }
  104. PackageRevision[] grabAll(Config config)
  105. {
  106. if (!exists("packagelist.json"))
  107. {
  108. return grabAllFresh(config);
  109. }
  110. auto js = "packagelist.json".readText.parseJSON;
  111. auto packageJsons = js["packages"].array;
  112. return packageJsons
  113. .filter!(x => x.type == JSON_TYPE.OBJECT)
  114. .map!(x => x.parsePackages(config))
  115. .joiner
  116. .array;
  117. }