Browse Source

First commit of backporter

dhasenan 4 years ago
commit
4da5b4fcee

+ 10
- 0
.gitignore View File

@@ -0,0 +1,10 @@
1
+*.swp
2
+/backporter
3
+*-test-library
4
+.dub
5
+packagelist.json
6
+*.csv
7
+dmd2*
8
+*.tar.xz
9
+log
10
+projects/

+ 6
- 0
dub.sdl View File

@@ -0,0 +1,6 @@
1
+name "backporter"
2
+description "A utility to automatically backport changes to old versions of DMD"
3
+authors "Neia Neutuladh"
4
+copyright "Copyright © 2018, Neia Neutuladh"
5
+license "Boost"
6
+dependency "scriptlike" version="~>0.10.2"

+ 6
- 0
dub.selections.json View File

@@ -0,0 +1,6 @@
1
+{
2
+	"fileVersion": 1,
3
+	"versions": {
4
+		"scriptlike": "0.10.2"
5
+	}
6
+}

+ 29
- 0
roadmap.todo View File

@@ -0,0 +1,29 @@
1
+# Backporter roadmap
2
+
3
+[-] Getting packages
4
+    [x] Grab the package list from dub
5
+    [x] Grab package details from dub
6
+    [x] Save dub package info locally so we don't annoy dub maintainers
7
+    [-] Checkout stuff from upstream
8
+        [x] git
9
+        [x] github
10
+        [ ] bitbucket
11
+    [x] Checkout the most recent release
12
+    [ ] Prefer releases with simple semver instead of eg 0.8.3-beta3
13
+[-] Getting DMD
14
+    [ ] Figure out what versions there are
15
+    [ ] Download and test all versions in a chosen range
16
+    [x] Download the requested version of DMD
17
+    [x] Extract to a directory
18
+    [x] Invoke dub from that directory
19
+    [x] Ensure the right DMD is being used
20
+[x] Building
21
+    [x] dub build
22
+    [x] dub test
23
+    [x] Record results as CSV
24
+[ ] Security
25
+    [ ] Run builds in a container with no permissions for security
26
+        [ ] Figure out how to share ~/.dub
27
+        [ ] Figure out how to properly constrain network
28
+        [ ] Figure out how to get results back
29
+    [ ] Time-limited builds

+ 7
- 0
source/app.d View File

@@ -0,0 +1,7 @@
1
+module app;
2
+
3
+int main(string[] args)
4
+{
5
+    import backporter.commandline;
6
+    return runCommandLine(args);
7
+}

+ 65
- 0
source/backporter/builder.d View File

@@ -0,0 +1,65 @@
1
+module backporter.builder;
2
+
3
+import backporter.git;
4
+import backporter.packagefinder;
5
+import std.stdio : File;
6
+import std.experimental.logger;
7
+
8
+struct Result
9
+{
10
+    string compilerVersion;
11
+    PackageRevision revision;
12
+    bool canCheckOut;
13
+    bool canBuild;
14
+    bool canTest;
15
+}
16
+
17
+Result build(string dubLoc, PackageRevision revision)
18
+{
19
+    Result result;
20
+    result.revision = revision;
21
+
22
+    if (!revision.checkout) return result;
23
+    result.canCheckOut = true;
24
+
25
+    cd(revision.pkg.checkoutDirectory);
26
+
27
+    // `dub build` includes app.d / main.d.
28
+    // `dub test` runs unittests.
29
+    // Without both, you can't be certain a package works.
30
+    // `dub test` isn't quite enough, and this doesn't get subpackages...but this is okay for starting
31
+    // out, at least.
32
+    auto buildResult = run(dubLoc, "build");
33
+    if (buildResult.status == 0)
34
+    {
35
+        result.canBuild = true;
36
+    }
37
+    auto testResult = run(dubLoc, "test");
38
+    if (testResult.status == 0)
39
+    {
40
+        result.canTest = true;
41
+    }
42
+    return result;
43
+}
44
+
45
+void buildAll(string compilerVersion, string dubLoc, Package[] pkgs, File outputFile)
46
+{
47
+    foreach (pkg; pkgs)
48
+    {
49
+        if (pkg.revisions.length == 0)
50
+        {
51
+            infof("package %s has no revisions; skipping", pkg.name);
52
+            continue;
53
+        }
54
+        auto res = build(dubLoc, pkg.revisions[$-1]);
55
+        res.compilerVersion = compilerVersion;
56
+        outputFile.writefln("%s,%s,%s,%s,%s,%s",
57
+                compilerVersion,
58
+                res.revision.pkg.name,
59
+                res.revision.id,
60
+                res.canCheckOut,
61
+                res.canBuild,
62
+                res.canTest);
63
+        outputFile.flush;
64
+    }
65
+}

+ 28
- 0
source/backporter/commandline.d View File

@@ -0,0 +1,28 @@
1
+module backporter.commandline;
2
+
3
+import backporter.builder;
4
+import backporter.dmd;
5
+import backporter.git;
6
+import backporter.packagefinder;
7
+
8
+int runCommandLine(string[] args)
9
+{
10
+    import std.stdio : File;
11
+    string release = "2.080.0";
12
+    string filename;
13
+
14
+    import std.getopt;
15
+    auto opties = getopt(
16
+            args,
17
+            "o|output", "the output file to write to", &outfilename,
18
+            "r|release", "the compiler release to use", &release
19
+            )
20
+
21
+    File outfile = File("results.csv", "w");
22
+
23
+    string dubLoc = download(release);
24
+    outfile.writeln("dmd,package,revision,checkout,build,test");
25
+    buildAll(release, dubLoc, grabAll, outfile);
26
+    return 0;
27
+}
28
+

+ 32
- 0
source/backporter/dmd.d View File

@@ -0,0 +1,32 @@
1
+module backporter.dmd;
2
+
3
+import std.experimental.logger;
4
+
5
+string download(string release)
6
+{
7
+    import std.string : format;
8
+    import std.net.curl : download;
9
+    import backporter.git : run, env, dataDir, cd;
10
+    import std.process : environment;
11
+    static import std.file;
12
+    import std.path : buildPath, absolutePath;
13
+
14
+    auto dir = "dmd" ~ release;
15
+    auto binDir = buildPath(dir, "dmd2/linux/bin64/").absolutePath;
16
+    std.file.mkdirRecurse(dir);
17
+    cd(dir);
18
+    scope (exit) cd(dataDir);
19
+    if (!std.file.exists(binDir))
20
+    {
21
+        auto url = "http://downloads.dlang.org/releases/2.x/%1$s/dmd.%1$s.linux.tar.xz".format(release);
22
+        infof("downloading dmd %s from %s", release, url);
23
+        auto xz = "dmd.tar.xz";
24
+        if (std.file.exists(xz))
25
+        std.file.remove(xz);
26
+        download(url, xz);
27
+        infof("downloaded, unpacking");
28
+        run("tar", "xf", xz);
29
+    }
30
+    env["PATH"] = binDir ~ ":" ~ environment["PATH"];
31
+    return binDir.buildPath("dub");
32
+}

+ 39
- 0
source/backporter/git.d View File

@@ -0,0 +1,39 @@
1
+module backporter.git;
2
+
3
+import std.experimental.logger;
4
+import std.process;
5
+
6
+static this()
7
+{
8
+    import std.file : getcwd;
9
+    import std.path : absolutePath, buildPath;
10
+    dataDir = buildPath(getcwd, "projects").absolutePath;
11
+}
12
+
13
+void cd(string dir)
14
+{
15
+    procDir = dir;
16
+    infof("cd %s", dir);
17
+}
18
+
19
+ProcessResult cmd(string name)(string[] args...)
20
+{
21
+    return run(["git", name] ~ args);
22
+}
23
+
24
+ProcessResult run(string[] args...)
25
+{
26
+    infof("running: %s", args);
27
+    return execute(args, env, Config.init, size_t.max, procDir);
28
+}
29
+
30
+alias clone = cmd!"clone";
31
+alias reset = cmd!"reset";
32
+
33
+private string procDir;
34
+
35
+string dataDir;
36
+string[string] env;
37
+
38
+import std.traits : ReturnType;
39
+alias ProcessResult = ReturnType!(std.process.execute);

+ 144
- 0
source/backporter/packagefinder.d View File

@@ -0,0 +1,144 @@
1
+module backporter.packagefinder;
2
+
3
+import std.algorithm, std.array;
4
+import backporter.git;
5
+import std.stdio;
6
+import std.file;
7
+import std.json;
8
+import std.path;
9
+import std.experimental.logger;
10
+
11
+class Package
12
+{
13
+    string name;
14
+    string gitUrl;
15
+    PackageRevision[] revisions;
16
+
17
+    this(JSONValue value)
18
+    {
19
+        name = value["name"].str;
20
+        auto repo = value["repository"];
21
+        switch (repo["kind"].str)
22
+        {
23
+            case "github":
24
+                gitUrl = "https://github.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str;
25
+                break;
26
+            default:
27
+                errorf("failed to parse repo info for project %s", name);
28
+                break;
29
+        }
30
+        foreach (revision; value["versions"].array)
31
+        {
32
+            if (!("version" in revision))
33
+            {
34
+                warningf("package %s contains a revision without a version name", name);
35
+                continue;
36
+            }
37
+            if (!("commitID" in revision))
38
+            {
39
+                warningf("package %s contains a revision without a commit id", name);
40
+                continue;
41
+            }
42
+            if (revision["version"].str[0] == '~') continue;
43
+            auto rev = new PackageRevision;
44
+            rev.pkg = this;
45
+            rev.commit = revision["commitID"].str;
46
+            rev.id = revision["version"].str;
47
+            rev.date = revision["date"].str;
48
+            revisions ~= rev;
49
+        }
50
+        import std.algorithm.sorting : sort;
51
+        import std.uni : icmp;
52
+        revisions.sort!((a, b) => a.date < b.date);
53
+    }
54
+
55
+    string checkoutDirectory()
56
+    {
57
+        return buildPath(dataDir, name);
58
+    }
59
+
60
+    bool ensureCloned()
61
+    {
62
+        import std.file : exists;
63
+        import backporter.git;
64
+        auto dir = checkoutDirectory;
65
+        if (dir.exists && chainPath(dir, ".git").exists) return true;
66
+        cd(dataDir);
67
+        return clone(gitUrl, name).status == 0;
68
+    }
69
+}
70
+
71
+class PackageRevision
72
+{
73
+    Package pkg;
74
+    string commit;
75
+    string id;
76
+    string date;
77
+
78
+    bool checkout()
79
+    {
80
+        if (!pkg.ensureCloned) return false;
81
+        cd(pkg.checkoutDirectory);
82
+        return reset("--hard", commit).status == 0;
83
+    }
84
+}
85
+
86
+JSONValue getPackageJson(string name)
87
+{
88
+    try
89
+    {
90
+        import std.net.curl : get;
91
+        infof("grabbing package %s", name);
92
+        auto js = get("https://code.dlang.org/api/packages/" ~ name ~ "/info").parseJSON;
93
+        return js;
94
+    }
95
+    catch (Exception e)
96
+    {
97
+        return JSONValue(null);
98
+    }
99
+}
100
+
101
+string[] packageNames()
102
+{
103
+    import std.net.curl : get;
104
+    infof("grabbing package list");
105
+    auto js = get("https://code.dlang.org/packages/index.json").parseJSON.array;
106
+    auto names = new string[js.length];
107
+    foreach (i, n; js)
108
+    {
109
+        names[i] = n.str;
110
+    }
111
+    names = names.sort.uniq.array;
112
+    infof("found %s packages", names.length);
113
+    return names;
114
+}
115
+
116
+Package[] grabAllFresh()
117
+{
118
+    auto names = packageNames;
119
+    auto packageJsons = names
120
+        .map!getPackageJson
121
+        .filter!(x => x.type == JSON_TYPE.OBJECT)
122
+        .array;
123
+    infof("downloading packages fresh");
124
+    JSONValue v;
125
+    v["packages"] = packageJsons;
126
+    auto f = File("packagelist.json", "w");
127
+    f.write(v.toString);
128
+    f.close;
129
+    return packageJsons.map!(x => new Package(x)).array;
130
+}
131
+
132
+Package[] grabAll()
133
+{
134
+    if (!exists("packagelist.json"))
135
+    {
136
+        return grabAllFresh;
137
+    }
138
+    auto js = "packagelist.json".readText.parseJSON;
139
+    auto packageJsons = js["packages"].array;
140
+    return packageJsons
141
+        .filter!(x => x.type == JSON_TYPE.OBJECT)
142
+        .map!(x => new Package(x))
143
+        .array;
144
+}