Neural net is In!

It's super slow. I suspect it's not learning properly.

source/app.d

@@ -1,22 +1,122 @@
 module app;
 
-import cosmic.ai;
 import cosmic.base;
 import cosmic.blocks;
 import cosmic.drawing;
-import cosmic.random;
-import cosmic.simple;
+import cosmic.ai.base;
+import cosmic.ai.nn;
+import cosmic.ai.random;
+import cosmic.ai.simple;
 
+import std.experimental.logger;
 import std.stdio;
 
-void main()
+int main(string[] args)
 {
-    auto board = new Board;
-    board.runGame(new Simple, new Random);
-    //mockup(board);
-    auto svg = new SvgSink();
-    board.draw(svg);
-    svg.write("board.svg");
+    globalLogLevel = LogLevel.info;
+    import std.getopt;
+    import etc.linux.memoryerror;
+    registerMemoryErrorHandler();
+
+    string command = "run";
+    string aia = "simple", aib = "random";
+    uint count = 1000;
+    string datafile = "learning.dat";
+    string nnfile = "nn.dat";
+    bool learn = true;
+
+    auto opts = getopt(
+            args,
+            "command|cmd", "what to do", &command,
+            "a", "which AI to use for player A", &aia,
+            "b", "which AI to use for player B", &aib,
+            "count|c", "how many games to play", &count,
+            "datafile|d", "where to read/store learning data", &datafile,
+            "nnfile|n", "where to read/store neural network data", &nnfile,
+            "learn", "whether to learn from playing", &learn);
+
+    if (opts.helpWanted)
+    {
+        defaultGetoptPrinter("Cosmic Blocks AI!!1", opts.options);
+        return 1;
+    }
+
+    if (command == "init")
+    {
+        auto nn = createNeuralNetwork(21, 11);
+        nn.serialize(nnfile);
+        return 0;
+    }
+
+    if (command == "bake")
+    {
+        bakeNeuralNetwork(nnfile, datafile);
+        return 0;
+    }
+
+    if (command == "script")
+    {
+        foreach (generation; 0..100)
+        {
+            NN.learningData = File(datafile, "ab");
+            auto a = new Neural();
+            auto b = new Neural();
+            auto board = new Board;
+            foreach (i; 0..count)
+            {
+                board.clear;
+                runGame(board, a, b);
+                infof("finished gen %s game %s; winner: %s", generation, i, board.winners);
+            }
+            SvgSink sink = new SvgSink;
+            board.draw(sink);
+            import std.format : format;
+            import std.file;
+            sink.write(format("board_gen%s.svg", generation));
+            infof("generation %s finished", generation);
+            NN.learningData.flush;
+            NN.learningData.close;
+            bakeNeuralNetwork(nnfile, datafile);
+            //std.file.remove(datafile);
+        }
+    }
+
+    AI create(string name)
+    {
+        switch (name)
+        {
+            case "nn":
+                return new Neural();
+            case "random":
+                return new Random();
+            case "simple":
+                return new Simple();
+            default:
+                throw new Exception("unrecognized ai name " ~ name);
+        }
+    }
+
+    if (command == "run")
+    {
+        NN.learningData = File(datafile, "ab");
+        auto a = create(aia);
+        auto b = create(aib);
+        auto board = new Board;
+        foreach (i; 0..count)
+        {
+            board.clear;
+            runGame(board, a, b);
+            tracef("finished game %s; winner: %s", i, board.winners);
+        }
+        SvgSink sink = new SvgSink;
+        board.draw(sink);
+        sink.write("board.svg");
+        NN.learningData.flush;
+        NN.learningData.close;
+        return 0;
+    }
+
+    return 0;
 }
 
 void mockup(Board board)
@@ -33,52 +133,46 @@ void mockup(Board board)
 
 void runGame(Board board, AI aia, AI aib)
 {
-    board.recalculateAccessibility;
-    foreach (block; board.blocks)
-    {
-        if (block.access != Player.none)
-        {
-            writefln("block %s accessible to %s", block.pos, block.access);
-        }
-    }
+    Board[] pastBoards;
     aia.init(board, Player.a);
     aib.init(board, Player.b);
     uint moves = 0;
     uint collisions = 0;
     while (true)
     {
+        board.tick;
         moves++;
-        if (moves > 300 || collisions > 10)
+        if (moves > 100 || collisions > 10)
         {
-            writefln("draw due to too many moves");
+            tracef("draw due to too many moves");
             break;
         }
         auto ma = aia.nextMove();
-        writefln("Player A plays: %s", ma);
+        tracef("Player A (%s) plays: %s", aia, ma);
         if (ma.forfeit)
         {
-            writeln("Player A forfeits");
+            //writeln("Player A forfeits");
             break;
         }
         assert(board[ma.pos].type is null, "tried to overwrite existing");
         if (board.winners != Player.none)
         {
-            writefln("winner: %s", board.winners);
+            tracef("winner: %s", board.winners);
             break;
         }
 
         auto mb = aib.nextMove();
-        writefln("Player B plays: %s", mb);
+        tracef("Player B (%s) plays: %s", aib, mb);
         if (mb.forfeit)
         {
-            writeln("Player B forfeits");
+            //writeln("Player B forfeits");
             break;
         }
         assert(board[mb.pos].type is null, "tried to overwrite existing");
 
         if (mb.pos == ma.pos)
         {
-            writefln("collision!");
+            tracef("collision!");
             board[ma.pos].type = BlockType.collision5;
             collisions++;
         }
@@ -88,26 +182,24 @@ void runGame(Board board, AI aia, AI aib)
             board[ma.pos].play(Player.a, ma.type, moves);
             board[mb.pos].play(Player.b, mb.type, moves);
         }
-        board.tick;
         if (board.winners != Player.none)
         {
-            writefln("winner: %s", board.winners);
+            tracef("winner: %s", board.winners);
             break;
         }
+        pastBoards ~= board.dup;
     }
-}
-
-Player winners(Board board)
-{
-    board.recalculateAccessibility;
-    Player w = Player.none;
-    if (board[board.startA].access & Player.b)
-    {
-        w |= Player.b;
-    }
-    if (board[board.startB].access & Player.a)
+    if (pastBoards.length > 0 && board.winners != Player.none)
     {
-        w |= Player.a;
+        foreach (i, b; pastBoards[1..$])
+        {
+            auto score = cast(int)(pastBoards.length - i);
+            if (board.winners == Player.a)
+            {
+                score *= -1;
+            }
+            nn.learn(b, score);
+        }
     }
-    return w;
 }
+

source/cosmic/ai/base.d

@@ -38,4 +38,6 @@ abstract class AI
     }
 
     abstract Move nextMove();
+
+    void finish(Player winner) {}
 }

source/cosmic/ai/nn.d

@@ -0,0 +1,201 @@
+module cosmic.ai.nn;
+
+import cosmic.ai.base;
+import cosmic.base;
+import cosmic.blocks;
+import vectorflow;
+import std.experimental.logger;
+import std.random;
+import std.file : exists, isFile;
+import std.stdio : File;
+
+class Neural : AI
+{
+    private static Mt19937 rnd;
+    static this()
+    {
+        rnd.seed(unpredictableSeed);
+    }
+
+    uint maxConsideredMoves = 5;
+    private int multiplier = 1;
+
+    override void init(Board board, Player player)
+    {
+        super.init(board, player);
+        if (nn is NN.init)
+        {
+            nn = NN(board.width, board.height);
+        }
+        if (player == Player.b)
+        {
+            multiplier = -1;
+        }
+    }
+
+    auto possibleMoves()
+    {
+        import std.algorithm, std.range;
+
+        auto emptySpaces = board.grid.data
+            .filter!(x => x.type is null)
+            .map!(x => x.pos)
+            .filter!(p => p.x >= board.startA.x - 1 && p.x <= board.startB.x + 1)
+            .filter!(p => p.y >= 2 && p.y <= board.height - 2)
+            ;
+        return cartesianProduct(emptySpaces, inventory)
+            .map!(x => Move(x[0], x[1]));
+    }
+
+    override Move nextMove()
+    {
+        import std.container.rbtree;
+        import std.stdio;
+
+        auto tmpBoard = board.dup;
+        auto tree = new RedBlackTree!ScoreMove;
+        uint count = 0;
+        foreach (move; possibleMoves)
+        {
+            count++;
+            tmpBoard[move.pos].type = move.type;
+            tmpBoard.recalculateAccessibility;
+            auto score = nn.classify(tmpBoard);
+            tmpBoard[move.pos].type = null;
+            auto sm = ScoreMove(move, score * multiplier, uniform(0, 1024, rnd));
+            tree.insert(sm);
+            if (tree.length > maxConsideredMoves)
+            {
+                tree.removeFront;
+            }
+        }
+        // We cleared all the new state from the tmpBoard, so we can store it for later learning.
+        if (tree.length == 0)
+        {
+            return Move.doForfeit;
+        }
+        tracef("nn: picked a move out of %s possibilites", count);
+        // Pick a random winner
+        if (tree.length == 1) return tree.front.move;
+        auto n = uniform(0, tree.length - 1, rnd);
+        foreach (i; 0..n) tree.removeFront;
+        return tree.front.move;
+    }
+
+    override void finish(Player winner)
+    {
+    }
+
+}
+
+static NN nn;
+
+struct NN
+{
+    static NeuralNet n;
+    static File learningData;
+    float[] buf;
+
+    static void load(string filename)
+    {
+        if (filename.exists && filename.isFile)
+        {
+            n = NeuralNet.deserialize(filename);
+        }
+    }
+
+    this(int width, int height)
+    {
+        buf = new float[width * height];
+        if (n is NeuralNet.init)
+        {
+            n = createNeuralNetwork(width, height);
+        }
+    }
+
+    void learn(Board board, int score)
+    {
+        board.serialize(buf);
+        ulong len = buf.length;
+        learningData.rawWrite((cast(ubyte*)&len)[0..len.sizeof]);
+        learningData.rawWrite((cast(ubyte*)&score)[0..score.sizeof]);
+        learningData.rawWrite(buf);
+        learningData.flush;
+    }
+
+    float classify(Board board)
+    {
+        board.serialize(buf);
+        auto v = n.predict(buf);
+        return v[0];
+    }
+}
+
+struct LearningInfo
+{
+    float label;
+    float[] features;
+}
+
+void bakeNeuralNetwork(string nnfile, string datafile)
+{
+    NeuralNet n;
+    try
+    {
+        infof("reading neural network from file %s", nnfile);
+        n = NeuralNet.deserialize(nnfile);
+        infof("finished reading neural network");
+    }
+    catch (Exception e)
+    {
+        infof("building new network due to error: %s", e.msg);
+        n = createNeuralNetwork(21, 11);
+    }
+
+    LearningInfo[] infos;
+    import std.file : read;
+    auto bytes = cast(ubyte[])datafile.read;
+    assert(bytes.ptr !is null, "read nothing from file??");
+    while (bytes.length > 12)
+    {
+        ulong length;
+        int score;
+        (cast(ubyte*)&length)[0..length.sizeof] = bytes[0..length.sizeof];
+        bytes = bytes[length.sizeof .. $];
+        (cast(ubyte*)&score)[0..score.sizeof] = bytes[0..score.sizeof];
+        bytes = bytes[score.sizeof .. $];
+        assert(length < 1000, "length of learning info too long");
+        auto features = (cast(float*)bytes.ptr)[0..length * float.sizeof];
+        infos ~= LearningInfo(cast(float)score, features);
+        bytes = bytes[length * float.sizeof .. $];
+    }
+
+    n.learn(infos, "multinomial", new ADAM(100, 0.0001, 200), true, 4);
+    n.serialize(nnfile);
+}
+
+NeuralNet createNeuralNetwork(int width, int height)
+{
+    return NeuralNet()
+        // First stack is inputs
+        .stack(DenseData(width * height))
+        // This is our first hidden layer
+        .stack(Linear(500))
+        .stack(DropOut(0.3))
+        .stack(Linear(1))  // We want one output
+        ;
+}
+
+struct ScoreMove
+{
+    Move move;
+    float score;
+    int rnd;
+
+    int opCmp(const ref ScoreMove other) inout
+    {
+        if (score < other.score) return -1;
+        if (score > other.score) return 1;
+        return rnd - other.rnd;
+    }
+}

source/cosmic/ai/nnsplay.d

@@ -0,0 +1,201 @@
+/**
+  * A neural network that attempts to answer directly which nodes will be in a positive solution.
+  */
+module cosmic.ai.nn;
+/+
+
+import cosmic.ai.base;
+import cosmic.base;
+import cosmic.blocks;
+import vectorflow;
+import std.experimental.logger;
+import std.random;
+import std.file : exists, isFile;
+import std.stdio : File;
+
+class NeuralSplay : AI
+{
+    private static Mt19937 rnd;
+    static this()
+    {
+        rnd.seed(unpredictableSeed);
+    }
+
+    uint maxConsideredMoves = 5;
+    private int multiplier = 1;
+
+    override void init(Board board, Player player)
+    {
+        super.init(board, player);
+        if (nn is NN.init)
+        {
+            nn = NN(board.width, board.height);
+        }
+        if (player == Player.b)
+        {
+            multiplier = -1;
+        }
+    }
+
+    auto possibleMoves()
+    {
+        import std.algorithm, std.range;
+
+        auto emptySpaces = board.grid.data
+            .filter!(x => x.type is null)
+            .map!(x => x.pos)
+            .filter!(p => p.x >= board.startA.x - 1 && p.x <= board.startB.x + 1)
+            .filter!(p => p.y >= 2 && p.y <= board.height - 2)
+            ;
+        return cartesianProduct(emptySpaces, inventory)
+            .map!(x => Move(x[0], x[1]));
+    }
+
+    override Move nextMove()
+    {
+        import std.container.rbtree;
+        import std.stdio;
+
+        auto buf = toNNInput(board);
+        auto result = nn.predict(buf);
+        // Take the top 5 values...
+        auto best = new RedBlackTree!ScoreMove;
+        foreach (i, score; result)
+        {
+            auto move = getMove(i);
+            if (!canMakeMove(move))
+            {
+                continue;
+            }
+            if (best.length < maxConsideredMoves)
+            {
+                // We just need something here.
+                best.insert(move);
+            }
+            if (score > best.back.score)
+            {
+                best.removeBack();
+                best.insert(ScoreMove(score, move));
+            }
+        }
+        if (best.length == 0)
+        {
+            return Move.doForfeit;
+        }
+        tracef("nn: picked a move out of %s possibilites", count);
+        // Pick a random winner
+        if (best.length == 1) return best.front.move;
+        auto n = uniform(0, best.length - 1, rnd);
+        foreach (i; 0..n) best.removeFront;
+        return best.front.move;
+    }
+
+    bool canMakeMove(Move move)
+    {
+        if (move.pos.x < 0 || move.pos.y < 0) return false;
+        if (move.pos.x >= board.width || move.pos.y >= board.height) return false;
+        if (board[move.pos].type !is null) return false;
+        // TODO inventory
+        return true;
+    }
+
+    Move getMove(size_t i)
+    {
+        // The result is the prediction about what is in the winning path.
+        // It's a flattened 3d array of floats, |BlockType.all| × width × height.
+        // Each row contains height × |BlockType.all| elements.
+        auto block = BlockType.all.length * board.height;
+        auto x = i / block;
+        auto rem = x % block;
+        auto y = rem / BlockType.all.length;
+        auto type = rem % BlockType.all.length;
+        return Move(Pos(x, y), BlockType.all[type]);
+    }
+
+    override void finish(Player winner)
+    {
+    }
+
+    static NeuralNet n;
+    static string learningFile = "learning_splay.dat";
+    static string brainFile = "nn_splay.dat";
+
+    static NeuralNet load()
+    {
+        if (n !is null) return n;
+        if (filename.exists && filename.isFile)
+        {
+            n = NeuralNet.deserialize(filename);
+        }
+        else
+        {
+            n = NeuralNet()
+                // First stack is inputs
+                .stack(DenseData(width * height * BlockType.all.length))
+                // This is our first hidden layer
+                .stack(Linear(500))
+                .stack(DropOut(0.3))
+                .stack(Linear(width * height * BlockType.all.length))
+                ;
+        }
+    }
+}
+
+static NN nn;
+
+struct NN
+{
+    static NeuralNet n;
+    static File learningData;
+    float[] buf;
+
+    static void load(string filename)
+    {
+        if (filename.exists && filename.isFile)
+        {
+            n = NeuralNet.deserialize(filename);
+        }
+    }
+
+    this(int width, int height)
+    {
+        buf = new float[width * height];
+        if (n is NeuralNet.init)
+        {
+            n = createNeuralNetwork(width, height);
+        }
+    }
+
+    static void learn(Board board, int score)
+    {
+        board.serialize(buf);
+        ulong len = buf.length;
+        learningData.rawWrite((cast(ubyte*)&len)[0..len.sizeof]);
+        learningData.rawWrite((cast(ubyte*)&score)[0..score.sizeof]);
+        learningData.rawWrite(buf);
+        learningData.flush;
+    }
+
+    float classify(Board board)
+    {
+        board.serialize(buf);
+        auto v = n.predict(buf);
+        return v[0];
+    }
+}
+
+struct ScoreMove
+{
+    Move move;
+    float score;
+    int rnd;
+
+    int opCmp(const ref ScoreMove other) inout
+    {
+        if (score < other.score) return -1;
+        if (score > other.score) return 1;
+        return rnd - other.rnd;
+    }
+}
+
++/

source/cosmic/ai/random.d

@@ -4,6 +4,7 @@ import cosmic.ai.base;
 import cosmic.base;
 import cosmic.blocks;
 import std.algorithm;
+import std.experimental.logger;
 import std.random;
 import std.range;
 
@@ -25,6 +26,7 @@ class Random : AI
             .walkLength;
         if (len == 0)
         {
+            tracef("nowhere to go from here; giving up");
             return Move.doForfeit;
         }
 

source/cosmic/ai/simple.d

@@ -50,7 +50,7 @@ class Simple : AI
     {
         // Find the open position closest to your opponent's position
         import std.algorithm, std.array;
-        infof("close gap for player %s", player);
+        //infof("close gap for player %s", player);
         auto options = board.blocks
             .filter!(x => (x.access & player) == player)
             .filter!(x => x.type is null)
@@ -59,7 +59,7 @@ class Simple : AI
         if (options.length == 0)
         {
             // TODO try to use a remove
-            info("failed to find a move: no open spaces");
+            //info("failed to find a move: no open spaces");
             return Move.doForfeit;
         }
         foreach (option; options)
@@ -81,7 +81,7 @@ class Simple : AI
                 return m;
             }
         }
-        info("nothing gets us closer; doing something random");
+        //info("nothing gets us closer; doing something random");
         Move m = {pos: options[0].pos, type: inventory[0]};
         return m;
     }

source/cosmic/base.d

@@ -49,6 +49,13 @@ struct Grid(T)
     {
         data[x + y * width] = value;
     }
+
+    typeof(this) dup()
+    {
+        auto next = this;
+        next.data = data.dup;
+        return next;
+    }
 }
 
 struct Pos

source/cosmic/blocks.d

@@ -10,6 +10,7 @@ final class Board
     Pos startA, startB;
     Grid!Block grid;
     alias grid this;
+    Player winners;
 
     this()
     {
@@ -19,6 +20,13 @@ final class Board
     this(int width, int height)
     {
         grid = Grid!Block(width, height);
+        clear();
+        recalculateAccessibility;
+    }
+
+    void clear()
+    {
+        winners = Player.none;
         foreach (x; 0..width)
         {
             foreach (y; 0..height)
@@ -26,14 +34,29 @@ final class Board
                 grid[x, y] = Block(null, Pos(x, y));
             }
         }
-
         // Starting positions
         auto halfHeight = height / 2;
         startA = Pos(halfHeight - 1, halfHeight);
         startB = Pos(width - halfHeight, halfHeight);
         grid[startA].type = BlockType.source;
         grid[startB].type = BlockType.source;
-        recalculateAccessibility;
+    }
+
+    Board dup()
+    {
+        auto d = new Board(grid.width, grid.height);
+        d.grid.data[] = grid.data[];
+        return d;
+    }
+
+    void serialize(ref float[] buf)
+    {
+        buf.length = grid.data.length;
+        import std.algorithm;
+        foreach (i, b; grid.data)
+        {
+            buf[i] = b.type is null ? 0f : cast(float)b.type.id;
+        }
     }
 
     string colorA = "red", colorB = "yellow", colorBoth = "orange", colorBg = "#E3EAEA";
@@ -48,12 +71,14 @@ final class Board
                 grid[x, y].tick;
             }
         }
-        /*
-        foreach (ref block; blocks)
+        if ((this[startA].access & Player.b) == Player.b)
+        {
+            winners |= Player.b;
+        }
+        if ((this[startB].access & Player.a) == Player.a)
         {
-            block.tick;
+            winners |= Player.a;
         }
-        */
     }
 
     void decayCollisions()
@@ -265,12 +290,14 @@ abstract class BlockType
     bool canCircle() @property { return circled != this; }
 
     string name;
+    char id;
     const Pos[] offsets;
     alias downstreamOffsets = offsets;
 
-    this(string nmae, const Pos[] offsets)
+    this(string name, char id, const Pos[] offsets)
     {
         this.name = name;
+        this.id = id;
         this.offsets = offsets;
     }
 
@@ -372,7 +399,7 @@ class BlockMine : BlockType
 {
     this()
     {
-        super("mine", null);
+        super("mine", 'm', null);
     }
 
     override void draw(SvgSink sink, Pos pos)
@@ -388,7 +415,7 @@ class BlockOX : BlockType
 {
     this()
     {
-        super("ox", [
+        super("ox", 'y', [
             Pos(-2, -2),
             Pos(-2,  2),
             Pos( 2, -2),
@@ -407,7 +434,7 @@ class BlockX : BlockType
 {
     this()
     {
-        super("x", [
+        super("x", 'x', [
             Pos(-1, -1),
             Pos(-1,  1),
             Pos( 1, -1),
@@ -427,7 +454,7 @@ class BlockPlus : BlockType
 {
     this()
     {
-        super("+", [
+        super("+", '+', [
             Pos(-1,  0),
             Pos( 1,  0),
             Pos( 0, -1),
@@ -447,7 +474,7 @@ class BlockOPlus : BlockType
 {
     this()
     {
-        super("o+", [
+        super("o+", '=', [
             Pos(-2,  0),
             Pos( 2,  0),
             Pos( 0, -2),
@@ -466,7 +493,7 @@ class BlockStar : BlockType
 {
     this()
     {
-        super("*", [
+        super("*", '*', [
             Pos(-1,  0),
             Pos( 1,  0),
             Pos( 0, -1),
@@ -491,7 +518,7 @@ class BlockOStar : BlockType
 {
     this()
     {
-        super("*", [
+        super("*", '/', [
             Pos(-2,  0),
             Pos( 2,  0),
             Pos( 0, -2),
@@ -516,7 +543,8 @@ class Arrow : BlockType
 
     this(string name, Pos toward)
     {
-        super(name, [toward]);
+        char id = cast(char)('A' + ((toward.x + 1) * 3 + toward.y + 1));
+        super(name, id, [toward]);
         this.toward = toward;
     }
 
@@ -553,7 +581,7 @@ class Collision : BlockType
     BlockType next;
     this(uint stage, BlockType next = null)
     {
-        super("collision" ~ stage.to!string, []);
+        super("collision" ~ stage.to!string, cast(char)('0' + stage), []);
         this.stage = stage;
         this.next = next;
         if (stage == 1)
@@ -568,7 +596,7 @@ class Collision : BlockType
 
     override void tick(ref Block block)
     {
-        infof("Collision tick: %s -> %s", this, next);
+        //infof("Collision tick: %s -> %s", this, next);
         block.type = next;
         block.placed = Player.none;
         block.turn = 0;
@@ -598,7 +626,7 @@ class BlockSource : BlockType
 {
     this()
     {
-        super("source", [
+        super("source", 'o', [
             Pos(-1,  0),
             Pos( 1,  0),
             Pos( 0, -1),