Add munching

munch/README.md

@@ -0,0 +1,3 @@
+Reads the game records from json, chooses which ones to use, splits them into training, validation, and test sets.
+
+Just cd here and run `bash process_games.sh`

munch/get_records.py

@@ -0,0 +1,8 @@
+import json
+import sys
+
+fname = sys.argv[1]
+data = json.load(open(fname))
+
+for game in data:
+    print(game["record"])

munch/process_games.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DATADIR="../data/"
+
+# get game records
+python get_records.py "$DATADIR"/bga-games-info.json > "$DATADIR"/records-all.txt
+
+# remove rows that appear more than once
+cat "$DATADIR"/records-all.txt | sort | uniq -u > "$DATADIR"/records-clean.txt
+
+# create train/valid/test data sets
+python split_records.py "$DATADIR"

munch/split_records.py

@@ -0,0 +1,21 @@
+import sys
+
+data_dir = sys.argv[1]
+
+train = open(data_dir + "/records-1-train.txt", "w")
+valid = open(data_dir + "/records-2-valid.txt", "w")
+test = open(data_dir + "/records-3-test.txt", "w")
+
+with open(data_dir + "/records-clean.txt") as f:
+    records = f.readlines()
+
+    counter = 0
+    for record in records:
+        counter += 1
+        fifth = counter % 5 + 1
+        if fifth == 5:
+            test.write(record)
+        elif fifth == 4:
+            valid.write(record)
+        else:
+            train.write(record)