Browse Source

Sqlite, multiple compiler / package versions

dhasenan 4 years ago
parent
commit
fbe690b526

+ 3
- 1
dub.sdl View File

@@ -3,4 +3,6 @@ description "A utility to automatically backport changes to old versions of DMD"
3 3
 authors "Neia Neutuladh"
4 4
 copyright "Copyright © 2018, Neia Neutuladh"
5 5
 license "Boost"
6
-dependency "scriptlike" version="~>0.10.2"
6
+dependency "scriptlike" version="0.10.2"
7
+dependency "d2sqlite3" version="0.16.2"
8
+dependency "datefmt" version="1.0.2"

+ 2
- 0
dub.selections.json View File

@@ -1,6 +1,8 @@
1 1
 {
2 2
 	"fileVersion": 1,
3 3
 	"versions": {
4
+		"d2sqlite3": "0.16.2",
5
+		"datefmt": "1.0.2",
4 6
 		"scriptlike": "0.10.2"
5 7
 	}
6 8
 }

+ 11
- 9
roadmap.todo View File

@@ -1,18 +1,16 @@
1 1
 # Basic functionality
2 2
 
3
-[-] Getting packages
3
+[x] Getting packages
4 4
     [x] Grab the package list from dub
5 5
     [x] Grab package details from dub
6 6
     [x] Save dub package info locally so we don't annoy dub maintainers
7
-    [-] Checkout stuff from upstream
8
-        [x] git
7
+    [x] Checkout stuff from upstream
8
+        [x] gitlab
9 9
         [x] github
10
-        [ ] bitbucket
10
+        [x] bitbucket
11 11
     [x] Checkout the most recent release
12 12
     [x] Prefer releases with simple semver instead of eg 0.8.3-beta3
13
-[-] Getting DMD
14
-    [ ] Figure out what versions there are
15
-    [ ] Download and test all versions in a chosen range
13
+[x] Getting DMD
16 14
     [x] Download the requested version of DMD
17 15
     [x] Extract to a directory
18 16
     [x] Invoke dub from that directory
@@ -22,11 +20,15 @@
22 20
     [x] dub test
23 21
     [x] Record results as CSV
24 22
     [x] Record whether the project uses anything deprecated
25
-    [ ] Build all versions
26
-    [ ] Don't re-build versions
23
+    [x] Build all versions
24
+    [x] Don't re-build versions too often
27 25
 [ ] Security
28 26
     [ ] Run builds in a container with no permissions for security
29 27
         [ ] Figure out how to share ~/.dub
30 28
         [ ] Figure out how to properly constrain network (no outgoing SMTP, for instance)
31 29
         [ ] Figure out how to get results back
32 30
     [x] Time-limited builds
31
+[x] Record results to sqlite
32
+[x] Test multiple DMD versions
33
+    [x] Figure out what versions there are
34
+    [x] Download and test all versions in a chosen range

+ 63
- 23
source/backporter/builder.d View File

@@ -1,42 +1,62 @@
1 1
 module backporter.builder;
2 2
 
3 3
 import backporter.git;
4
-import backporter.packagefinder;
5 4
 import backporter.core;
6 5
 import std.stdio : File;
7 6
 import std.experimental.logger;
7
+import std.file;
8
+import std.path;
8 9
 
9
-struct Result
10
+bool checkout(PackageRevision pkg, string dir)
10 11
 {
11
-    string compilerVersion;
12
-    PackageRevision revision;
13
-    bool canCheckOut;
14
-    bool canBuild;
15
-    bool canTest;
16
-    bool hasDeprecations;
12
+    if (!dir.exists || !chainPath(dir, ".git").exists)
13
+    {
14
+        auto parent = dirName(dir);
15
+        mkdirRecurse(parent);
16
+        cd(parent);
17
+        if (clone(pkg.gitUrl, pkg.packageName).status != 0) return false;
18
+    }
19
+    cd(dir);
20
+    return reset("--hard", pkg.commitId).status == 0;
17 21
 }
18 22
 
19
-Result build(const ref Config config, PackageRevision revision)
23
+Build build(Config config, PackageRevision revision)
20 24
 {
21
-    Result result;
22
-    result.revision = revision;
25
+    Build result;
26
+    result.revisionId = revision.revisionId;
27
+    result.packageName = revision.packageName;
28
+    result.compilerRelease = config.compilerRelease;
29
+    static import datefmt;
30
+    import std.datetime.systime : Clock;
31
+    result.time = datefmt.format(Clock.currTime, datefmt.ISO8601FORMAT);
23 32
 
24
-    if (!revision.checkout) return result;
33
+    auto dir = config.dataDir.buildPath(revision.packageName);
34
+    if (!checkout(revision, dir)) return result;
25 35
     result.canCheckOut = true;
26 36
 
27
-    cd(revision.pkg.checkoutDirectory);
37
+    cd(dir);
28 38
 
29 39
     // `dub build` includes app.d / main.d.
30 40
     // `dub test` runs unittests.
31 41
     // Without both, you can't be certain a package works.
32 42
     // `dub test` isn't quite enough, and this doesn't get subpackages...but this is okay for
33 43
     // starting out, at least.
34
-    auto buildResult = run(config.buildTimeout, config.dubLoc, "build");
44
+    auto buildResult = run(
45
+            config.buildTimeout,
46
+            config.dubLoc,
47
+            "build",
48
+            "--compiler=" ~ config.dmdLoc
49
+            );
35 50
     if (buildResult.status == 0)
36 51
     {
37 52
         result.canBuild = true;
38 53
     }
39
-    auto testResult = run(config.buildTimeout, config.dubLoc, "test");
54
+    auto testResult = run(
55
+            config.buildTimeout,
56
+            config.dubLoc,
57
+            "test",
58
+            "--compiler=" ~ config.dmdLoc
59
+            );
40 60
     if (testResult.status == 0)
41 61
     {
42 62
         result.canTest = true;
@@ -47,17 +67,13 @@ Result build(const ref Config config, PackageRevision revision)
47 67
     return result;
48 68
 }
49 69
 
50
-void buildAll(ref Config config, Package[] pkgs)
70
+void buildAll(Config config, PackageRevision[] pkgs)
51 71
 {
52 72
     foreach (pkg; pkgs)
53 73
     {
54
-        if (pkg.revisions.length == 0)
55
-        {
56
-            infof("package %s has no revisions; skipping", pkg.name);
57
-            continue;
58
-        }
59
-        auto res = build(config, pkg.revisions[$-1]);
60
-        res.compilerVersion = config.compilerRelease;
74
+        auto res = build(config, pkg);
75
+        config.record(res);
76
+        /*
61 77
         config.outfile.writefln("%s,%s,%s,%s,%s,%s,%s",
62 78
                 config.compilerRelease,
63 79
                 res.revision.pkg.name,
@@ -67,5 +83,29 @@ void buildAll(ref Config config, Package[] pkgs)
67 83
                 res.canTest,
68 84
                 res.hasDeprecations);
69 85
         config.outfile.flush;
86
+        */
87
+    }
88
+}
89
+
90
+void exerciseRelease(Config config, string compilerRelease)
91
+{
92
+    import backporter.dmd;
93
+    config.compilerRelease = compilerRelease;
94
+    downloadCompiler(config);
95
+    auto remaining = config.countRemainingPackages(compilerRelease);
96
+    while (true)
97
+    {
98
+        auto pkg = config.nextPackage(config.compilerRelease);
99
+        if (pkg == PackageRevision.init)
100
+        {
101
+            break;
102
+        }
103
+        infof("dmd %s package %s.%s (%s remaining)",
104
+                config.compilerRelease,
105
+                pkg.packageName,
106
+                pkg.revisionId,
107
+                remaining);
108
+        remaining--;
109
+        config.record(build(config, pkg));
70 110
     }
71 111
 }

+ 90
- 11
source/backporter/commandline.d View File

@@ -5,12 +5,15 @@ import backporter.dmd;
5 5
 import backporter.git;
6 6
 import backporter.packagefinder;
7 7
 import backporter.core;
8
+import std.range;
9
+import std.experimental.logger;
8 10
 
9 11
 int runCommandLine(string[] args)
10 12
 {
11 13
     import std.stdio : File;
12
-    Config config;
14
+    auto config = new Config;
13 15
     string blacklistPath;
16
+    bool verbose;
14 17
 
15 18
     import std.getopt;
16 19
     auto opties = getopt(
@@ -18,25 +21,101 @@ int runCommandLine(string[] args)
18 21
             "o|output", "the output file to write to", &config.outpath,
19 22
             "r|release", "the compiler release to use", &config.compilerRelease,
20 23
             "w|workdir", "the directory to work in", &config.dataDir,
21
-            "b|blacklist", "file containing project blacklists", &blacklistPath
24
+            "b|blacklist", "file containing project blacklists", &blacklistPath,
25
+            "v|verbose", "enable verbose logging", &verbose,
26
+            "c|version-count", "how many revisions per package to compile",
27
+                &config.revisionCount
22 28
             );
29
+
30
+    if (opties.helpWanted)
31
+    {
32
+        defaultGetoptPrinter("backporter: misnamed tester for all dub packages",
33
+                opties.options);
34
+        return 1;
35
+    }
36
+    if (verbose)
37
+    {
38
+        globalLogLevel = LogLevel.trace;
39
+    }
40
+    else
41
+    {
42
+        globalLogLevel = LogLevel.info;
43
+    }
44
+
45
+    config.init;
23 46
     if (blacklistPath)
24 47
     {
25 48
         import std.array : array;
26 49
         config.blacklistedProjects = File(blacklistPath, "r").byLineCopy.array;
27 50
     }
28 51
 
29
-    if (opties.helpWanted)
52
+    auto pkgs = grabAll(config);
53
+    config.importPackages(pkgs);
54
+    auto compilerReleases = [
55
+		"2.082.0",
56
+		"2.081.2",
57
+		"2.081.1",
58
+		"2.081.0",
59
+		"2.080.1",
60
+		"2.080.0",
61
+		"2.079.1",
62
+		"2.079.0",
63
+		"2.078.3",
64
+		"2.078.2",
65
+		"2.078.1",
66
+		"2.078.0",
67
+		"2.077.1",
68
+		"2.077.0",
69
+		"2.076.1",
70
+		"2.076.0",
71
+		"2.075.1",
72
+		"2.075.0",
73
+		"2.074.1",
74
+		"2.074.0",
75
+		"2.073.2",
76
+		"2.073.1",
77
+		"2.073.0",
78
+		"2.072.2",
79
+		"2.072.1",
80
+		"2.072.0",
81
+		"2.071.2",
82
+		"2.071.1",
83
+		"2.071.0",
84
+		"2.070.2",
85
+		"2.070.1",
86
+		"2.070.0",
87
+		"2.069.2",
88
+		"2.069.1",
89
+		"2.069.0",
90
+		"2.068.2",
91
+		"2.068.1",
92
+		"2.068.0",
93
+		"2.067.1",
94
+		"2.067.0",
95
+		"2.066.1",
96
+		"2.066.0",
97
+		"2.065.0",
98
+    ];
99
+    // What order should we test things in?
100
+    // We eventually want to test everything. But we want each prefix to be as
101
+    // useful as possible. So we do every tenth release (about every 9 months)
102
+    // until we get to the end, at which point we back up and try the next
103
+    // offset for the stride.
104
+    // If you interrupt in the middle, we've got a decent range of releases,
105
+    // hopefully, and the more recent ones are more thoroughly covered.
106
+    //
107
+    // Ideally, we'd divide-and-conquer: the most recent, then the last, then
108
+    // the midpoints, and repeat the midpoints until finished.
109
+    auto strideLength = 10;
110
+    infof("have %s compiler releases and %s package revisions to test",
111
+            compilerReleases.length, pkgs.length);
112
+    foreach (i; 0..strideLength)
30 113
     {
31
-        defaultGetoptPrinter("backporter: misnamed tester for all dub packages", opties.options);
32
-        return 1;
114
+        foreach (release; compilerReleases[i .. $].stride(strideLength))
115
+        {
116
+            exerciseRelease(config, release);
117
+        }
33 118
     }
34
-
35
-    config.outfile = File("results.csv", "w");
36
-
37
-    downloadCompiler(config);
38
-    config.outfile.writeln("dmd,package,revision,checkout,build,test,deprecations");
39
-    buildAll(config, grabAll());
40 119
     return 0;
41 120
 }
42 121
 

+ 190
- 5
source/backporter/core.d View File

@@ -1,16 +1,201 @@
1 1
 module backporter.core;
2 2
 
3 3
 import core.time;
4
+import d2sqlite3;
4 5
 import std.stdio : File;
6
+import std.datetime.systime : SysTime;
5 7
 
6
-struct Config
8
+// TODO: rename this class 'God'
9
+class Config
7 10
 {
8
-    string outpath = "results.csv";
9
-    File outfile;
10
-    bool useDocker = true;
11
+    this()
12
+    {
13
+        import std.path : absolutePath;
14
+        outpath = "results.sqlite".absolutePath;
15
+        dataDir = "projects".absolutePath;
16
+    }
17
+
18
+    string outpath;
19
+    string dataDir;
11 20
     string compilerRelease = "2.080.0";
12 21
     string dubLoc;
22
+    string dmdLoc;
23
+    string fallbackDub;
13 24
     string[] blacklistedProjects;
14
-    string dataDir = "projects";
15 25
     Duration buildTimeout = 5.minutes;
26
+    bool useDocker = true;
27
+    ulong revisionCount = 3;
28
+
29
+    void init()
30
+    {
31
+        import std.path : absolutePath;
32
+        db = Database(outpath.absolutePath);
33
+        db.run(`CREATE TABLE IF NOT EXISTS builds
34
+                (
35
+                 time TEXT PRIMARY KEY,
36
+                 compilerRelease TEXT NOT NULL,
37
+                 packageName TEXT NOT NULL,
38
+                 revisionId TEXT NOT NULL,
39
+                 skipped BOOLEAN NOT NULL,
40
+                 canCheckOut BOOLEAN NOT NULL,
41
+                 canBuild BOOLEAN NOT NULL,
42
+                 canTest BOOLEAN NOT NULL,
43
+                 hasDeprecations BOOLEAN NOT NULL
44
+                )`);
45
+        db.run(`CREATE TABLE IF NOT EXISTS revisions
46
+                (
47
+                 packageName TEXT NOT NULL,
48
+                 revisionId TEXT NOT NULL,
49
+                 gitUrl TEXT NOT NULL,
50
+                 commitId TEXT NOT NULL,
51
+                 lastBuilt TEXT NULL,
52
+                 PRIMARY KEY (packageName, revisionId)
53
+                )`);
54
+        addRecord = db.prepare(`
55
+                INSERT INTO builds
56
+                (time, compilerRelease, packageName, revisionId, skipped, canCheckOut, canBuild,
57
+                 canTest, hasDeprecations)
58
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
59
+                `);
60
+        updateBuildTime = db.prepare(`
61
+                UPDATE revisions
62
+                SET lastBuilt = $3
63
+                WHERE packageName = $1 AND revisionId = $2
64
+                `);
65
+        nextPackages = db.prepare(`
66
+                SELECT * FROM revisions
67
+                ORDER BY lastBuilt ASC, packageName DESC, revisionId DESC
68
+                LIMIT ?
69
+                `);
70
+        addPackage = db.prepare(`
71
+                INSERT INTO revisions
72
+                (packageName, revisionId, gitUrl, commitId, lastBuilt)
73
+                VALUES
74
+                (?, ?, ?, ?, ?)
75
+                `);
76
+        nextPackageSingle = db.prepare(`
77
+                SELECT * FROM revisions
78
+                WHERE NOT EXISTS
79
+                (
80
+                 SELECT * FROM builds
81
+                 WHERE compilerRelease = ?
82
+                 AND packageName = revisions.packageName
83
+                 AND revisionId = revisions.revisionId
84
+                )
85
+                ORDER BY revisionId DESC
86
+                LIMIT 1
87
+                `);
88
+        remainingPackageCount = db.prepare(`
89
+                SELECT COUNT(*) FROM revisions
90
+                WHERE NOT EXISTS
91
+                (
92
+                 SELECT * FROM builds
93
+                 WHERE compilerRelease = ?
94
+                 AND packageName = revisions.packageName
95
+                 AND revisionId = revisions.revisionId
96
+                )
97
+                `);
98
+
99
+        {
100
+            import scriptlike;
101
+            fallbackDub = runCollect("which dub").strip;
102
+        }
103
+    }
104
+
105
+    ~this()
106
+    {
107
+        db.close;
108
+    }
109
+
110
+    void record(Build result)
111
+    {
112
+        addRecord.inject(result);
113
+        updateBuildTime.inject(result.packageName, result.revisionId, result.time);
114
+    }
115
+
116
+    void importPackages(PackageRevision[] revisions)
117
+    {
118
+        db.run("begin");
119
+        scope (exit) db.run("commit");
120
+        db.run("delete from revisions");
121
+        foreach (rev; revisions)
122
+        {
123
+            addPackage.inject(rev);
124
+        }
125
+    }
126
+
127
+    PackageRevision nextPackage(string compilerRelease)
128
+    {
129
+        nextPackageSingle.reset;
130
+        nextPackageSingle.bind(1, compilerRelease);
131
+        auto result = nextPackageSingle.execute;
132
+        if (result.empty) return PackageRevision.init;
133
+        return result.front.as!PackageRevision;
134
+    }
135
+
136
+    PackageRevision[] fetchPackages(uint count)
137
+    {
138
+        import std.algorithm.iteration : map;
139
+        import std.array : array;
140
+        nextPackages.reset;
141
+        nextPackages.bind(1, count);
142
+        auto result = nextPackages.execute;
143
+        return result.map!(x => x.as!PackageRevision).array;
144
+    }
145
+
146
+    ulong countRemainingPackages(string compilerRelease)
147
+    {
148
+        remainingPackageCount.reset;
149
+        remainingPackageCount.bind(1, compilerRelease);
150
+        return remainingPackageCount.execute.oneValue!ulong;
151
+    }
152
+
153
+private:
154
+    File _outfile;
155
+    Database db;
156
+    Statement addRecord;
157
+    Statement nextPackages;
158
+    Statement updateBuildTime;
159
+    Statement addPackage;
160
+    Statement nextPackageSingle;
161
+    Statement remainingPackageCount;
162
+}
163
+
164
+struct PackageRevision
165
+{
166
+    string packageName;
167
+    string revisionId;
168
+    string gitUrl;
169
+    string commitId;
170
+    string lastBuilt;  // ISO timestamp
171
+}
172
+
173
+struct Build
174
+{
175
+    /// When this build happened.
176
+    string time;
177
+
178
+    /// The compiler version we used.
179
+    string compilerRelease;
180
+
181
+    /// The package that we built.
182
+    string packageName;
183
+
184
+    /// The ID of the revision that we built.
185
+    string revisionId;
186
+
187
+    /// Whether we skipped this for some reason.
188
+    bool skipped;
189
+
190
+    /// Whether we could check this package out.
191
+    bool canCheckOut;
192
+
193
+    /// Whether we could build this package.
194
+    bool canBuild;
195
+
196
+    /// Whether the tests passed.
197
+    bool canTest;
198
+
199
+    /// Whether there's any deprecated code in this.
200
+    bool hasDeprecations;
16 201
 }

+ 25
- 7
source/backporter/dmd.d View File

@@ -5,7 +5,20 @@ import std.experimental.logger;
5 5
 
6 6
 void downloadCompiler(ref Config config)
7 7
 {
8
-    config.dubLoc = download(config.compilerRelease);
8
+    static import std.file;
9
+    import std.path : buildPath;
10
+    auto binLoc = download(config.compilerRelease);
11
+    config.dmdLoc = buildPath(binLoc, "dmd");
12
+
13
+    auto dubLoc = buildPath(binLoc, "dub");
14
+    if (std.file.exists(dubLoc))
15
+    {
16
+        config.dubLoc = dubLoc;
17
+    }
18
+    else
19
+    {
20
+        config.dubLoc = config.fallbackDub;
21
+    }
9 22
 }
10 23
 
11 24
 private string download(string release)
@@ -17,8 +30,10 @@ private string download(string release)
17 30
     static import std.file;
18 31
     import std.path : buildPath, absolutePath;
19 32
 
20
-    auto dir = "dmd" ~ release;
33
+    auto dir = ("dmd" ~ release).absolutePath;
21 34
     auto binDir = buildPath(dir, "dmd2/linux/bin64/").absolutePath;
35
+    auto dmd = binDir.buildPath("dmd");
36
+    if (std.file.exists(dmd)) return binDir;
22 37
     std.file.mkdirRecurse(dir);
23 38
     cd(dir);
24 39
     scope (exit) cd(dataDir);
@@ -26,13 +41,16 @@ private string download(string release)
26 41
     {
27 42
         auto url = "http://downloads.dlang.org/releases/2.x/%1$s/dmd.%1$s.linux.tar.xz".format(release);
28 43
         infof("downloading dmd %s from %s", release, url);
29
-        auto xz = "dmd.tar.xz";
30
-        if (std.file.exists(xz))
31
-        std.file.remove(xz);
44
+        auto xz = dir.buildPath("dmd.tar.xz");
45
+        if (std.file.exists(xz)) std.file.remove(xz);
32 46
         download(url, xz);
33 47
         infof("downloaded, unpacking");
34
-        run("tar", "xf", xz);
48
+        auto res = run("tar", "xf", xz);
49
+        if (res.status != 0)
50
+        {
51
+            throw new Exception("failed to download/extract " ~ release ~ " from " ~ url);
52
+        }
35 53
     }
36 54
     env["PATH"] = binDir ~ ":" ~ environment["PATH"];
37
-    return binDir.buildPath("dub");
55
+    return binDir;
38 56
 }

+ 11
- 4
source/backporter/git.d View File

@@ -16,16 +16,21 @@ static this()
16 16
 void cd(string dir)
17 17
 {
18 18
     procDir = dir;
19
-    infof("cd %s", dir);
19
+    tracef("cd %s", dir);
20 20
 }
21 21
 
22 22
 ProcessResult cmd(string name)(string[] args...)
23 23
 {
24
-    return run(["git", name] ~ args);
24
+    // If a project has been deleted, github (at least) asks you to log in,
25
+    // because it might instead have been made private.
26
+    // 30 seconds should be plenty to get the data when it's available and
27
+    // not terribly long to wait otherwise.
28
+    return run(30.seconds, ["git", name] ~ args);
25 29
 }
26 30
 
27 31
 ProcessResult run(string[] args...)
28 32
 {
33
+    tracef("running: %s", args);
29 34
     auto start = Clock.currTime;
30 35
     auto p = execute(args, env, Config.init, size_t.max, procDir);
31 36
     auto end = Clock.currTime;
@@ -34,7 +39,7 @@ ProcessResult run(string[] args...)
34 39
 
35 40
 ProcessResult run(Duration maxTime, string[] args...)
36 41
 {
37
-    infof("running: %s", args);
42
+    tracef("running@%s: %s", procDir, args);
38 43
     ProcessResult result;
39 44
 
40 45
     import std.path : buildPath;
@@ -45,7 +50,9 @@ ProcessResult run(Duration maxTime, string[] args...)
45 50
     auto pid = spawnProcess(
46 51
             args,
47 52
             // stdin / stdout / stderr
48
-            File("/dev/null", "r"), File("/dev/null", "w"), childStderr,
53
+            File("/dev/null", "r"),
54
+            File(buildPath(procDir, ".stdout"), "w"),
55
+            childStderr,
49 56
             env,
50 57
             Config.none,
51 58
             procDir);

+ 61
- 88
source/backporter/packagefinder.d View File

@@ -1,6 +1,7 @@
1 1
 module backporter.packagefinder;
2 2
 
3 3
 import std.algorithm, std.array;
4
+import backporter.core;
4 5
 import backporter.git;
5 6
 import std.stdio;
6 7
 import std.file;
@@ -8,89 +9,6 @@ import std.json;
8 9
 import std.path;
9 10
 import std.experimental.logger;
10 11
 
11
-class Package
12
-{
13
-    string name;
14
-    string gitUrl;
15
-    PackageRevision[] revisions;
16
-    PackageRevision[] betas;
17
-
18
-    this(JSONValue value)
19
-    {
20
-        name = value["name"].str;
21
-        auto repo = value["repository"];
22
-        switch (repo["kind"].str)
23
-        {
24
-            case "github":
25
-                gitUrl = "https://github.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str;
26
-                break;
27
-            default:
28
-                errorf("failed to parse repo info for project %s", name);
29
-                break;
30
-        }
31
-        foreach (revision; value["versions"].array)
32
-        {
33
-            if (!("version" in revision))
34
-            {
35
-                warningf("package %s contains a revision without a version name", name);
36
-                continue;
37
-            }
38
-            if (!("commitID" in revision))
39
-            {
40
-                warningf("package %s contains a revision without a commit id", name);
41
-                continue;
42
-            }
43
-            if (revision["version"].str[0] == '~') continue;
44
-            auto rev = new PackageRevision;
45
-            rev.pkg = this;
46
-            rev.commit = revision["commitID"].str;
47
-            rev.id = revision["version"].str;
48
-            rev.date = revision["date"].str;
49
-            if (rev.id.canFind('-'))
50
-            {
51
-                betas ~= rev;
52
-            }
53
-            else
54
-            {
55
-                revisions ~= rev;
56
-            }
57
-        }
58
-        import std.algorithm.sorting : sort;
59
-        import std.uni : icmp;
60
-        revisions.sort!((a, b) => a.date < b.date);
61
-    }
62
-
63
-    string checkoutDirectory()
64
-    {
65
-        return buildPath(dataDir, name);
66
-    }
67
-
68
-    bool ensureCloned()
69
-    {
70
-        import std.file : exists;
71
-        import backporter.git;
72
-        auto dir = checkoutDirectory;
73
-        if (dir.exists && chainPath(dir, ".git").exists) return true;
74
-        cd(dataDir);
75
-        return clone(gitUrl, name).status == 0;
76
-    }
77
-}
78
-
79
-class PackageRevision
80
-{
81
-    Package pkg;
82
-    string commit;
83
-    string id;
84
-    string date;
85
-
86
-    bool checkout()
87
-    {
88
-        if (!pkg.ensureCloned) return false;
89
-        cd(pkg.checkoutDirectory);
90
-        return reset("--hard", commit).status == 0;
91
-    }
92
-}
93
-
94 12
 JSONValue getPackageJson(string name)
95 13
 {
96 14
     try
@@ -121,7 +39,61 @@ string[] packageNames()
121 39
     return names;
122 40
 }
123 41
 
124
-Package[] grabAllFresh()
42
+auto parsePackages(JSONValue value, Config config)
43
+{
44
+    PackageRevision[] revisions;
45
+    auto name = value["name"].str;
46
+    auto repo = value["repository"];
47
+    string gitUrl;
48
+
49
+    switch (repo["kind"].str)
50
+    {
51
+        case "github":
52
+            gitUrl = "https://github.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str;
53
+            break;
54
+        case "gitlab":
55
+            gitUrl = "https://gitlab.com/" ~ repo["owner"].str ~ "/" ~ repo["project"].str ~ ".git";
56
+            break;
57
+        case "bitbucket":
58
+            gitUrl = "https://bitbucket.org/" ~ repo["owner"].str ~ "/" ~ repo["project"].str ~ ".git";
59
+            break;
60
+        default:
61
+            errorf("failed to parse repo info for project %s", name);
62
+            break;
63
+    }
64
+
65
+    auto v = value["versions"].array;
66
+    v.sort!((a, b) => (a["date"].str > b["date"].str));
67
+    foreach (revision; value["versions"].array)
68
+    {
69
+        if (!("version" in revision))
70
+        {
71
+            warningf("package %s contains a revision without a version name", name);
72
+            continue;
73
+        }
74
+        if (!("commitID" in revision))
75
+        {
76
+            warningf("package %s contains a revision without a commit id", name);
77
+            continue;
78
+        }
79
+        auto id = revision["version"].str;
80
+        if (id.canFind('-')) continue;
81
+        if (id.canFind('~')) continue;
82
+        PackageRevision rev;
83
+        rev.commitId = revision["commitID"].str;
84
+        rev.revisionId = id;
85
+        rev.packageName = name;
86
+        rev.gitUrl = gitUrl;
87
+        revisions ~= rev;
88
+    }
89
+    if (revisions.length <= config.revisionCount)
90
+    {
91
+        return revisions;
92
+    }
93
+    return revisions[0..config.revisionCount];
94
+}
95
+
96
+PackageRevision[] grabAllFresh(Config config)
125 97
 {
126 98
     auto names = packageNames;
127 99
     auto packageJsons = names
@@ -134,19 +106,20 @@ Package[] grabAllFresh()
134 106
     auto f = File("packagelist.json", "w");
135 107
     f.write(v.toString);
136 108
     f.close;
137
-    return packageJsons.map!(x => new Package(x)).array;
109
+    return packageJsons.map!(x => x.parsePackages(config)).joiner.array;
138 110
 }
139 111
 
140
-Package[] grabAll()
112
+PackageRevision[] grabAll(Config config)
141 113
 {
142 114
     if (!exists("packagelist.json"))
143 115
     {
144
-        return grabAllFresh;
116
+        return grabAllFresh(config);
145 117
     }
146 118
     auto js = "packagelist.json".readText.parseJSON;
147 119
     auto packageJsons = js["packages"].array;
148 120
     return packageJsons
149 121
         .filter!(x => x.type == JSON_TYPE.OBJECT)
150
-        .map!(x => new Package(x))
122
+        .map!(x => x.parsePackages(config))
123
+        .joiner
151 124
         .array;
152 125
 }