First commit of backporter

.gitignore

@@ -0,0 +1,10 @@
+*.swp
+/backporter
+*-test-library
+.dub
+packagelist.json
+*.csv
+dmd2*
+*.tar.xz
+log
+projects/

dub.sdl

@@ -0,0 +1,6 @@
+name "backporter"
+description "A utility to automatically backport changes to old versions of DMD"
+authors "Neia Neutuladh"
+copyright "Copyright © 2018, Neia Neutuladh"
+license "Boost"
+dependency "scriptlike" version="~>0.10.2"

dub.selections.json

@@ -0,0 +1,6 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"scriptlike": "0.10.2"
+	}
+}

roadmap.todo

@@ -0,0 +1,29 @@
+# Backporter roadmap
+
+[-] Getting packages
+    [x] Grab the package list from dub
+    [x] Grab package details from dub
+    [x] Save dub package info locally so we don't annoy dub maintainers
+    [-] Checkout stuff from upstream
+        [x] git
+        [x] github
+        [ ] bitbucket
+    [x] Checkout the most recent release
+    [ ] Prefer releases with simple semver instead of eg 0.8.3-beta3
+[-] Getting DMD
+    [ ] Figure out what versions there are
+    [ ] Download and test all versions in a chosen range
+    [x] Download the requested version of DMD
+    [x] Extract to a directory
+    [x] Invoke dub from that directory
+    [x] Ensure the right DMD is being used
+[x] Building
+    [x] dub build
+    [x] dub test
+    [x] Record results as CSV
+[ ] Security
+    [ ] Run builds in a container with no permissions for security
+        [ ] Figure out how to share ~/.dub
+        [ ] Figure out how to properly constrain network
+        [ ] Figure out how to get results back
+    [ ] Time-limited builds

source/app.d

@@ -0,0 +1,7 @@
+module app;
+
+int main(string[] args)
+{
+    import backporter.commandline;
+    return runCommandLine(args);
+}

source/backporter/builder.d

@@ -0,0 +1,65 @@
+module backporter.builder;
+
+import backporter.git;
+import backporter.packagefinder;
+import std.stdio : File;
+import std.experimental.logger;
+
+struct Result
+{
+    string compilerVersion;
+    PackageRevision revision;
+    bool canCheckOut;
+    bool canBuild;
+    bool canTest;
+}
+
+Result build(string dubLoc, PackageRevision revision)
+{
+    Result result;
+    result.revision = revision;
+
+    if (!revision.checkout) return result;
+    result.canCheckOut = true;
+
+    cd(revision.pkg.checkoutDirectory);
+
+    // `dub build` includes app.d / main.d.
+    // `dub test` runs unittests.
+    // Without both, you can't be certain a package works.
+    // `dub test` isn't quite enough, and this doesn't get subpackages...but this is okay for starting
+    // out, at least.
+    auto buildResult = run(dubLoc, "build");
+    if (buildResult.status == 0)
+    {
+        result.canBuild = true;
+    }
+    auto testResult = run(dubLoc, "test");
+    if (testResult.status == 0)
+    {
+        result.canTest = true;
+    }
+    return result;
+}
+
+void buildAll(string compilerVersion, string dubLoc, Package[] pkgs, File outputFile)
+{
+    foreach (pkg; pkgs)
+    {
+        if (pkg.revisions.length == 0)
+        {
+            infof("package %s has no revisions; skipping", pkg.name);
+            continue;
+        }
+        auto res = build(dubLoc, pkg.revisions[$-1]);
+        res.compilerVersion = compilerVersion;
+        outputFile.writefln("%s,%s,%s,%s,%s,%s",
+                compilerVersion,
+                res.revision.pkg.name,
+                res.revision.id,
+                res.canCheckOut,
+                res.canBuild,
+                res.canTest);
+        outputFile.flush;
+    }
+}

source/backporter/commandline.d

@@ -0,0 +1,28 @@
+module backporter.commandline;
+
+import backporter.builder;
+import backporter.dmd;
+import backporter.git;
+import backporter.packagefinder;
+
+int runCommandLine(string[] args)
+{
+    import std.stdio : File;
+    string release = "2.080.0";
+    string filename;
+
+    import std.getopt;
+    auto opties = getopt(
+            args,
+            "o|output", "the output file to write to", &outfilename,
+            "r|release", "the compiler release to use", &release
+            )
+
+    File outfile = File("results.csv", "w");
+
+    string dubLoc = download(release);
+    outfile.writeln("dmd,package,revision,checkout,build,test");
+    buildAll(release, dubLoc, grabAll, outfile);
+    return 0;
+}
+

source/backporter/dmd.d

@@ -0,0 +1,32 @@
+module backporter.dmd;
+
+import std.experimental.logger;
+
+string download(string release)
+{
+    import std.string : format;
+    import std.net.curl : download;
+    import backporter.git : run, env, dataDir, cd;
+    import std.process : environment;
+    static import std.file;
+    import std.path : buildPath, absolutePath;
+
+    auto dir = "dmd" ~ release;
+    auto binDir = buildPath(dir, "dmd2/linux/bin64/").absolutePath;
+    std.file.mkdirRecurse(dir);
+    cd(dir);
+    scope (exit) cd(dataDir);
+    if (!std.file.exists(binDir))
+    {
+        auto url = "http://downloads.dlang.org/releases/2.x/%1$s/dmd.%1$s.linux.tar.xz".format(release);
+        infof("downloading dmd %s from %s", release, url);
+        auto xz = "dmd.tar.xz";
+        if (std.file.exists(xz))
+        std.file.remove(xz);
+        download(url, xz);
+        infof("downloaded, unpacking");
+        run("tar", "xf", xz);
+    }
+    env["PATH"] = binDir ~ ":" ~ environment["PATH"];
+    return binDir.buildPath("dub");
+}

source/backporter/git.d

@@ -0,0 +1,39 @@
+module backporter.git;
+
+import std.experimental.logger;
+import std.process;
+
+static this()
+{
+    import std.file : getcwd;
+    import std.path : absolutePath, buildPath;
+    dataDir = buildPath(getcwd, "projects").absolutePath;
+}
+
+void cd(string dir)
+{
+    procDir = dir;
+    infof("cd %s", dir);
+}
+
+ProcessResult cmd(string name)(string[] args...)
+{
+    return run(["git", name] ~ args);
+}
+
+ProcessResult run(string[] args...)
+{
+    infof("running: %s", args);
+    return execute(args, env, Config.init, size_t.max, procDir);
+}
+
+alias clone = cmd!"clone";
+alias reset = cmd!"reset";
+
+private string procDir;
+
+string dataDir;
+string[string] env;
+
+import std.traits : ReturnType;
+alias ProcessResult = ReturnType!(std.process.execute);

source/backporter/packagefinder.d

@@ -0,0 +1,144 @@
+module backporter.packagefinder;
+
+import std.algorithm, std.array;
+import backporter.git;
+import std.stdio;
+import std.file;
+import std.json;
+import std.path;
+import std.experimental.logger;
+
+class Package
+{
+    string name;
+    string gitUrl;
+    PackageRevision[] revisions;
+
+    this(JSONValue value)
+    {
+        name = value["name"].str;
+        auto repo = value["repository"];
+        switch (repo["kind"].str)
+        {
+            case "github":
+                gitUrl = "https://github.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str;
+                break;
+            default:
+                errorf("failed to parse repo info for project %s", name);
+                break;
+        }
+        foreach (revision; value["versions"].array)
+        {
+            if (!("version" in revision))
+            {
+                warningf("package %s contains a revision without a version name", name);
+                continue;
+            }
+            if (!("commitID" in revision))
+            {
+                warningf("package %s contains a revision without a commit id", name);
+                continue;
+            }
+            if (revision["version"].str[0] == '~') continue;
+            auto rev = new PackageRevision;
+            rev.pkg = this;
+            rev.commit = revision["commitID"].str;
+            rev.id = revision["version"].str;
+            rev.date = revision["date"].str;
+            revisions ~= rev;
+        }
+        import std.algorithm.sorting : sort;
+        import std.uni : icmp;
+        revisions.sort!((a, b) => a.date < b.date);
+    }
+
+    string checkoutDirectory()
+    {
+        return buildPath(dataDir, name);
+    }
+
+    bool ensureCloned()
+    {
+        import std.file : exists;
+        import backporter.git;
+        auto dir = checkoutDirectory;
+        if (dir.exists && chainPath(dir, ".git").exists) return true;
+        cd(dataDir);
+        return clone(gitUrl, name).status == 0;
+    }
+}
+
+class PackageRevision
+{
+    Package pkg;
+    string commit;
+    string id;
+    string date;
+
+    bool checkout()
+    {
+        if (!pkg.ensureCloned) return false;
+        cd(pkg.checkoutDirectory);
+        return reset("--hard", commit).status == 0;
+    }
+}
+
+JSONValue getPackageJson(string name)
+{
+    try
+    {
+        import std.net.curl : get;
+        infof("grabbing package %s", name);
+        auto js = get("https://code.dlang.org/api/packages/" ~ name ~ "/info").parseJSON;
+        return js;
+    }
+    catch (Exception e)
+    {
+        return JSONValue(null);
+    }
+}
+
+string[] packageNames()
+{
+    import std.net.curl : get;
+    infof("grabbing package list");
+    auto js = get("https://code.dlang.org/packages/index.json").parseJSON.array;
+    auto names = new string[js.length];
+    foreach (i, n; js)
+    {
+        names[i] = n.str;
+    }
+    names = names.sort.uniq.array;
+    infof("found %s packages", names.length);
+    return names;
+}
+
+Package[] grabAllFresh()
+{
+    auto names = packageNames;
+    auto packageJsons = names
+        .map!getPackageJson
+        .filter!(x => x.type == JSON_TYPE.OBJECT)
+        .array;
+    infof("downloading packages fresh");
+    JSONValue v;
+    v["packages"] = packageJsons;
+    auto f = File("packagelist.json", "w");
+    f.write(v.toString);
+    f.close;
+    return packageJsons.map!(x => new Package(x)).array;
+}
+
+Package[] grabAll()
+{
+    if (!exists("packagelist.json"))
+    {
+        return grabAllFresh;
+    }
+    auto js = "packagelist.json".readText.parseJSON;
+    auto packageJsons = js["packages"].array;
+    return packageJsons
+        .filter!(x => x.type == JSON_TYPE.OBJECT)
+        .map!(x => new Package(x))
+        .array;
+}