Browse Source

Reworked some frontend stuff?

Rearranging, better parsing, some CLI adjustments, working on test
runner
dhasenan 1 year ago
parent
commit
6e769f09c8

BIN
dpetalc/petalc View File


+ 32
- 7
dpetalc/src/petal/cli.d View File

@@ -4,14 +4,16 @@ int runMain(string[] args)
4 4
 {
5 5
     import std.getopt;
6 6
     string outfile = "a.out";
7
-    bool saveTemps;
7
+    string saveTemps;
8
+    string stage;
8 9
     bool verbose;
9 10
     auto opts = getopt(
10 11
         args,
11 12
         config.caseSensitive,
12 13
         config.bundling,
13 14
         "o|outfile", "output filename", &outfile,
14
-        "s|save-temps", "save intermediate files (LLVM IR etc)", &saveTemps,
15
+        "s|save-temps", "save intermediate files (LLVM IR etc) to the given directory", &saveTemps,
16
+        "stage", "how much compilation to do (mainly for tests)", &stage,
15 17
         "v|verbose", "verbose logging", &verbose);
16 18
 
17 19
     if (opts.helpWanted)
@@ -20,20 +22,41 @@ int runMain(string[] args)
20 22
         return 0;
21 23
     }
22 24
 
23
-    import petal.lexer : Lexer;
24
-    import petal.parser : PetalParser;
25
+    import petal.frontend.core : Intern;
26
+    import petal.frontend.errors : ErrorRecorder;
27
+    import petal.frontend.lexer : Lexer;
28
+    import petal.frontend.parser : Parser;
29
+    import petal.frontend.ast : ModuleDecl;
25 30
 
31
+    auto errors = new ErrorRecorder;
32
+
33
+    ModuleDecl[] toCompile;
26 34
     foreach (file; args)
27 35
     {
28 36
         // Should probably use a memory-mapped file?
29 37
         import std.file : readText;
30
-        import petal.errors;
31
-        auto lex = Lexer(file, file.readText, new ErrorRecorder, new Intern);
32
-        auto p = new Parser(lex);
38
+        import petal.frontend.errors;
39
+        auto lex = Lexer(file, file.readText, errors, new Intern);
40
+        if (stage == "lex")
41
+        {
42
+            while (!lex.empty) lex.popFront;
43
+            continue;
44
+        }
45
+        auto p = new Parser(lex, errors);
33 46
         auto m = p.parseModule;
47
+        toCompile ~= m;
34 48
         import std.stdio;
35 49
         writefln("parsed %s", m.name);
36 50
     }
51
+    if (stage == "parse" || stage == "lex")
52
+    {
53
+        return errors.exitCode;
54
+    }
55
+
56
+    foreach (m; toCompile)
57
+    {
58
+        // TODO compile
59
+    }
37 60
 
38 61
     return 0;
39 62
 }
@@ -42,6 +65,8 @@ int main(string[] args)
42 65
 {
43 66
     version (unittest)
44 67
     {
68
+        import std.stdio : writeln;
69
+        writeln("All tests have passed successfully. (I'm guessing here.)");
45 70
         return 0;
46 71
     }
47 72
     else

+ 0
- 1
dpetalc/src/petal/conv.d View File

@@ -1 +0,0 @@
1
-module petal.conv;

dpetalc/src/petal/ast.d → dpetalc/src/petal/frontend/ast.d View File

@@ -1,9 +1,9 @@
1
-module petal.ast;
1
+module petal.frontend.ast;
2 2
 
3
-import petal.core;
4
-import petal.errors;
5
-import petal.lexer;
6
-import petal.types;
3
+import petal.frontend.core;
4
+import petal.frontend.errors;
5
+import petal.frontend.lexer;
6
+import petal.frontend.types;
7 7
 
8 8
 class Using : Decl
9 9
 {
@@ -80,14 +80,13 @@ class OverloadSet: Decl
80 80
 }
81 81
 class TypeDecl: Decl
82 82
 {
83
-    Type type;
84 83
     Decl[] children;
85 84
 }
86 85
 class VarDecl: Decl
87 86
 {
88
-    Type type;
87
+    Expr typeExpr;
89 88
     string name;
90
-    Expr init;
89
+    Expr initExpr;
91 90
     bool defaultInit;
92 91
     // Specific to VarDecls that are aggregate fields: the offset of the field
93 92
     int offset = -1;
@@ -95,9 +94,11 @@ class VarDecl: Decl
95 94
 class FuncDecl: Decl
96 95
 {
97 96
     FuncArg[] args;
98
-    BlockStatement body_;
97
+    BlockStatement fnBody;
99 98
     Type returnType;
99
+    Expr returnTypeExpr;
100 100
     bool isExtern;
101
+    bool forwardDecl;
101 102
 
102 103
     bool match(CallExpr call, ErrorRecorder errors)
103 104
     {

+ 1
- 0
dpetalc/src/petal/frontend/conv.d View File

@@ -0,0 +1 @@
1
+module petal.frontend.conv;

dpetalc/src/petal/core.d → dpetalc/src/petal/frontend/core.d View File

@@ -1,5 +1,6 @@
1
-module petal.core;
1
+module petal.frontend.core;
2 2
 
3
+import petal.frontend.types;
3 4
 import petal.tagged;
4 5
 
5 6
 class Intern
@@ -24,40 +25,6 @@ struct File
24 25
     string fullPath;
25 26
     string data;
26 27
 }
27
-class FileTable
28
-{
29
-    string basePath;
30
-    this(string basePath)
31
-    {
32
-        this.basePath = basePath;
33
-    }
34
-
35
-    File opIndex(string path)
36
-    {
37
-        path = absolutePath(path, basePath);
38
-        if (auto fp = path in byPath)
39
-        {
40
-            return *fp;
41
-        }
42
-        File f;
43
-        numFiles++;
44
-        f.id = numFiles;
45
-        f.fullPath = path;
46
-        auto m = new MmFile(path, MmFile.Mode.read, 0, null);
47
-        f.data = cast(string)m[];
48
-        byPath[f.fullPath] = f;
49
-        byId[f.id] = f;
50
-        mapped[f.id] = m;
51
-        return f;
52
-    }
53
-
54
-private:
55
-    import std.mmfile;
56
-    uint numFiles;
57
-    File[string] byPath;
58
-    File[uint] byId;
59
-    MmFile[uint] mapped;
60
-}
61 28
 
62 29
 /**
63 30
  * A location in a file.
@@ -138,3 +105,8 @@ abstract class Type : SemNode
138 105
         return other is this;
139 106
     }
140 107
 }
108
+
109
+static this()
110
+{
111
+    BuiltinType.init();
112
+}

dpetalc/src/petal/errors.d → dpetalc/src/petal/frontend/errors.d View File

@@ -1,6 +1,6 @@
1
-module petal.errors;
1
+module petal.frontend.errors;
2 2
 
3
-import petal.core;
3
+import petal.frontend.core;
4 4
 
5 5
 struct ErrorType
6 6
 {
@@ -32,12 +32,38 @@ struct ErrorType
32 32
 
33 33
 class ErrorRecorder
34 34
 {
35
-    void report(Loc loc, ErrorType type, string message = null) {}
36
-    void report(SemNode node, ErrorType type, string message = null) {}
37
-    void report(SemNode node, SemNode related, ErrorType type, string message = null) {}
35
+    int exitCode;
36
+    void report(Loc loc, ErrorType type, string message = null)
37
+    {
38
+        import std.stdio;
39
+        stderr.writefln(
40
+                "%s: P%s: %s",
41
+                loc,
42
+                type.id,
43
+                message == null ? type.message : message);
44
+        exitCode = 1;
45
+    }
46
+
47
+    void report(SemNode node, ErrorType type, string message = null)
48
+    {
49
+        report(node.loc, type, message);
50
+    }
51
+
52
+    void report(SemNode node, SemNode related, ErrorType type, string message = null)
53
+    {
54
+        report(node, type, message);
55
+        import std.stdio;
56
+        stderr.writefln("  see also: %s", related.loc);
57
+    }
58
+
38 59
     template reportf(TArgs...)
39 60
     {
40 61
         import std.format : format;
62
+        void reportf(Loc loc, ErrorType type, string fmt, TArgs args)
63
+        {
64
+            report(loc, type, format(fmt, args));
65
+        }
66
+
41 67
         void reportf(SemNode node, ErrorType type, string fmt, TArgs args)
42 68
         {
43 69
             report(node, null, type, format(fmt, args));

dpetalc/src/petal/idchars.d → dpetalc/src/petal/frontend/idchars.d View File

@@ -1,4 +1,4 @@
1
-module petal.idchars;
1
+module petal.frontend.idchars;
2 2
 
3 3
 /* This was autogenerated with a tool:
4 4
 

dpetalc/src/petal/lexer.d → dpetalc/src/petal/frontend/lexer.d View File

@@ -1,7 +1,7 @@
1
-module petal.lexer;
1
+module petal.frontend.lexer;
2 2
 
3
-import petal.core;
4
-import petal.errors;
3
+import petal.frontend.core;
4
+import petal.frontend.errors;
5 5
 import std.string : startsWith, indexOf;
6 6
 
7 7
 struct Lexer
@@ -38,6 +38,11 @@ struct Lexer
38 38
         front = doLex();
39 39
     }
40 40
 
41
+    bool empty()
42
+    {
43
+        return front.tok == Tok.eof;
44
+    }
45
+
41 46
     Token doLex()
42 47
     {
43 48
         if (remaining.length == 0)
@@ -149,7 +154,7 @@ struct Lexer
149 154
                 return v;
150 155
             }
151 156
         }
152
-        errors.report(loc, "unterminated string literal");
157
+        errors.report(loc, ErrorType.syntaxError, "unterminated string literal");
153 158
         auto v = Token(loc, Tok.str, remaining);
154 159
         eat(remaining.length);
155 160
         return v;
@@ -171,7 +176,7 @@ struct Lexer
171 176
             }
172 177
             if (end == 0)
173 178
             {
174
-                errorf("unexpected token: invalid character %s", c);
179
+                errors.reportf(loc, ErrorType.syntaxError, "unexpected token: invalid character %s", c);
175 180
                 end = remaining.length;
176 181
             }
177 182
             goto foundEnd;
@@ -197,7 +202,7 @@ foundEnd:
197 202
 
198 203
     bool isValidIdentEnd(dchar c)
199 204
     {
200
-        import petal.idchars;
205
+        import petal.frontend.idchars;
201 206
         // Fast path for mostly-ASCII text.
202 207
         if (c == (c & 0x7F))
203 208
         {
@@ -239,8 +244,11 @@ foundEnd:
239 244
 
240 245
     unittest
241 246
     {
242
-        auto l = new Lexer("", "");
243
-        import petal.idchars;
247
+        import std.format : format;
248
+        auto err = new ErrorRecorder;
249
+        auto intern = new Intern;
250
+        auto l = Lexer("", "", err, intern);
251
+        import petal.frontend.idchars;
244 252
 
245 253
         // Exhaustive testing.
246 254
         foreach (dchar c; 'a' .. 'z'+1)

dpetalc/src/petal/parser.d → dpetalc/src/petal/frontend/parser.d View File

@@ -1,9 +1,9 @@
1
-module petal.parser;
1
+module petal.frontend.parser;
2 2
 
3
-import petal.ast;
4
-import petal.core;
5
-import petal.errors;
6
-import petal.lexer;
3
+import petal.frontend.ast;
4
+import petal.frontend.core;
5
+import petal.frontend.errors;
6
+import petal.frontend.lexer;
7 7
 
8 8
 class Parser
9 9
 {
@@ -53,6 +53,7 @@ class Parser
53 53
         if (decl is null && attrs.length > 0)
54 54
         {
55 55
             errors.report(start, ErrorType.syntaxError, "expected declaration following attribute");
56
+            synchronize!([Tok.eos, Tok.cbrace]);
56 57
         }
57 58
         if (decl !is null)
58 59
         {
@@ -65,6 +66,97 @@ class Parser
65 66
         return decl;
66 67
     }
67 68
 
69
+    void synchronize(Tok[] toks)()
70
+    {
71
+        while (!peek(Tok.eof))
72
+        {
73
+            static foreach (t; toks)
74
+            {
75
+                if (peek(t)) return;
76
+            }
77
+        }
78
+    }
79
+
80
+    VarDecl parseVarDecl()
81
+    {
82
+        eat(Tok.let);
83
+        auto decl = new VarDecl;
84
+        decl.name = eat(Tok.ident).data;
85
+        if (!peek(Tok.assign) && !peek(Tok.bang) && !peek(Tok.eos))
86
+        {
87
+            decl.typeExpr = parseExpr();
88
+        }
89
+        if (peek(Tok.assign))
90
+        {
91
+            eat(Tok.assign);
92
+            decl.initExpr = parseExpr;
93
+            eat(Tok.eos);
94
+            return decl;
95
+        }
96
+        if (peek(Tok.bang))
97
+        {
98
+            eat(Tok.bang);
99
+            decl.defaultInit = true;
100
+            return decl;
101
+        }
102
+        eat(Tok.eos);
103
+        return decl;
104
+    }
105
+
106
+    FuncDecl parseFuncDecl()
107
+    {
108
+        eat(Tok.fn);
109
+        auto decl = new FuncDecl;
110
+        decl.name = eat(Tok.ident).data;
111
+        while (!peek(Tok.eof))
112
+        {
113
+            if (peek(Tok.arrow))
114
+            {
115
+                eat(Tok.arrow);
116
+                decl.returnTypeExpr = parseExpr;
117
+                decl.fnBody = parseBlockStatement;
118
+                return decl;
119
+            }
120
+            if (peek(Tok.obrace))
121
+            {
122
+                decl.fnBody = parseBlockStatement;
123
+                return decl;
124
+            }
125
+            if (peek(Tok.eos))
126
+            {
127
+                decl.forwardDecl = true;
128
+                return decl;
129
+            }
130
+            decl.args ~= parseFuncArg;
131
+        }
132
+        return decl;
133
+    }
134
+
135
+    FuncArg parseFuncArg()
136
+    {
137
+        return null;
138
+    }
139
+
140
+    Expr parseExpr()
141
+    {
142
+        return null;
143
+    }
144
+
145
+    Statement parseStatement()
146
+    {
147
+        synchronize!([Tok.eos, Tok.cbrace]);
148
+        tryEat(Tok.eos);
149
+        return null;
150
+    }
151
+
152
+    BlockStatement parseBlockStatement()
153
+    {
154
+        eat(Tok.obrace);
155
+        auto block = new BlockStatement();
156
+        while (!peek(Tok.cbrace)) block.statements ~= parseStatement();
157
+        return block;
158
+    }
159
+
68 160
     Using parseUsing()
69 161
     {
70 162
         eat(Tok.using);
@@ -75,7 +167,7 @@ class Parser
75 167
             u.selective = true;
76 168
             while (!tryEat(Tok.eos) && !peek(Tok.eof))
77 169
             {
78
-                u.expose ~= eat(Tok.ident);
170
+                u.expose ~= eat(Tok.ident).data;
79 171
                 if (!tryEat(Tok.comma) && !peek(Tok.eos))
80 172
                 {
81 173
                     errors.report(
@@ -87,7 +179,7 @@ class Parser
87 179
         }
88 180
         else if (tryEat(Tok.as))
89 181
         {
90
-            u.renamed = eat(Tok.ident);
182
+            u.renamed = eat(Tok.ident).data;
91 183
         }
92 184
         eat(Tok.eos);
93 185
         return u;
@@ -126,7 +218,6 @@ class Parser
126 218
             return p;
127 219
         }
128 220
         errors.reportf(p.loc, ErrorType.syntaxError, "expected %s", t);
129
-        // TODO throw exception, synchronize, etc
130 221
         assert(false);
131 222
     }
132 223
 

dpetalc/src/petal/types.d → dpetalc/src/petal/frontend/types.d View File

@@ -1,6 +1,6 @@
1
-module petal.types;
1
+module petal.frontend.types;
2 2
 
3
-import petal.core;
3
+import petal.frontend.core;
4 4
 
5 5
 struct BuiltinType
6 6
 {
@@ -14,4 +14,7 @@ struct BuiltinType
14 14
         stringLiteral,
15 15
         boolean
16 16
         ;
17
+    static void init()
18
+    {
19
+    }
17 20
 }

+ 109
- 1
dpetalc/testrunner.d View File

@@ -1,6 +1,114 @@
1 1
 module testrunner;
2 2
 
3
-void main(string[] args)
3
+import std.algorithm;
4
+import std.experimental.logger;
5
+import std.file;
6
+import std.process;
7
+import std.stdio;
8
+
9
+enum Op
10
+{
11
+    lex,
12
+    parse,
13
+    run
14
+}
15
+
16
+struct Error
17
+{
18
+    uint line;
19
+    uint id;
20
+    string message;
21
+
22
+    bool opEquals(const ref Error other) const
23
+    {
24
+        if (line != other.line) return false;
25
+        if (id != other.id) return false;
26
+        if (message.length && other.message.length)
27
+        {
28
+            if (message != other.message) return false;
29
+        }
30
+        return true;
31
+    }
32
+}
33
+
34
+struct Test
4 35
 {
36
+    string shortPath;
37
+    Op op;
38
+    Error[] expected;
39
+    uint expectedExitCode;
40
+}
41
+
42
+struct Execution
43
+{
44
+    Duration time;
45
+    int exitCode;
46
+    string stdout;
47
+    string stderr;
48
+}
49
+
50
+struct Compiler
51
+{
52
+    string[] baseArgs;
53
+    this(string path, string[] extraArgs)
54
+    {
55
+        mkdirRecurse("tmp");
56
+        baseArgs = [path, "--save-temps", "tmp"] ~ extraArgs;
57
+    }
58
+
59
+    Execution run(const Test test) const
60
+    {
61
+        import std.process;
62
+        auto stdin = pipe();
63
+        auto stdout = pipe();
64
+        auto stderr = pipe();
65
+        auto args = baseArgs ~ test.fullPath;
66
+        final switch (test.op) with (Op)
67
+        {
68
+            case lex:
69
+                args ~= "--stage=lex";
70
+                break;
71
+            case parse:
72
+                args ~= "--stage=parse";
73
+                break;
74
+            default:
75
+                break;
76
+        }
77
+        auto pid = spawnProcess(path, args, );
78
+    }
79
+}
80
+
81
+int main(string[] args)
82
+{
83
+    string testdir = "../jpetalc/src/test/petal";
84
+    string compiler = "./petalc";
85
+    uint timeoutMillis = 500;
86
+
87
+    auto opts = getopt(
88
+        args,
89
+        "t|testdir", "directory to look for tests", &testdir,
90
+        "c|compiler", "compiler path", &compiler);
91
+    if (opts.helpWanted)
92
+    {
93
+        defaultGetoptPrinter("Run Petal's test suite", opts.options);
94
+        return 0;
95
+    }
5 96
 
97
+    // * Locate our test cases
98
+    // * Figure out what we need to do with each of them
99
+    auto tests = readTests(testdir);
100
+    TestResult[] testResults;
101
+    foreach (test; tests)
102
+    {
103
+        // * Run the compiler on them
104
+        // * Make sure the compiler finishes in a reasonable amount of time
105
+        // * Make sure the output is what we expect
106
+        // * Record the results
107
+        testResults ~= runCompiler(compiler, test, timeout);
108
+    }
109
+    // * Show them in a reasonable format (console + HTML report)
110
+    buildHtmlReport(testResults);
111
+    // * Exit code appropriately
112
+    if (testResults.any!(x => x.failed)) return 1;
113
+    return 0;
6 114
 }

+ 13
- 1
typesystem.md View File

@@ -7,6 +7,8 @@ and accepts being incomplete.
7 7
 The type system operates at compile-time insofar as possible. If something could just as easily be a
8 8
 compile-time error as a runtime error, it should be a compile-time error.
9 9
 
10
+# Basic types
11
+
10 12
 ## Use of uninitialized variables
11 13
 Using uninitialized variables is not allowed.
12 14
 
@@ -383,11 +385,21 @@ An aggregate may declare that it adheres to a concept.
383 385
 ### Operator overloading
384 386
 User-defined types can overload operators. Please do not abuse this feature.
385 387
 * indexing: `index` function for retrieving a value, `setIndex` for changing a value, `range` for
386
-  slicing.
388
+  slicing, `end` for `$`.
387 389
 * math: `add`, `subtract`, `multiply`, `divide`, `modulus`, `exponent`
388 390
 * bitwise: `bitAnd`, `bitOr`, `bitXor`
389 391
 * bitwise shifts: `rightShift`, `leftShift`, `rightRotate`, `leftRotate`
390 392
 * iteration: `range` with no arguments to return a range.
391 393
 
392 394
 ## Mutability
395
+
393 396
 By default, local variables are mutable, and everything else is immutable.
397
+
398
+# Algebraic data types
399
+
400
+Petal supports type algebra.
401
+
402
+    # s is either an int or a string
403
+    let int | string s = "";
404
+    # t is an int and a string: a tuple.
405
+    let int & string t = 0 & "";