Benchmark Configurations

benchmarks.py

@@ -3,7 +3,7 @@ import environment_wrapper as ew
 import gym
 from carla_environment import CarlaEnvironment
 import copy
-from tqdm import trange
+import threading
 
 c = ew.Config()
 
@@ -22,6 +22,11 @@ smallNet.name = 'SmallNet'
 smallNet.net_layout = [128, 32]
 smallNet.conf_to_name()
 
+smallNetDeep = copy.deepcopy(c)
+smallNetDeep.name = 'SmallNetDepp'
+smallNetDeep.net_layout = [128, 32, 32]
+smallNetDeep.conf_to_name()
+
 normalNet = copy.deepcopy(c)
 normalNet.name = 'NormalNet'
 normalNet.net_layout = [256, 128]
@@ -49,10 +54,41 @@ deepNet.name = 'DeppNet'
 deepNet.net_layout = [256, 128, 128]
 deepNet.conf_to_name()
 
-# configurations = [smallNet, normalNet ]
-# configurations = [normalSlowDecay, normalSlowLearn]
-configurations = [largeNet, deepNet]
+littleNet = copy.deepcopy(c)
+littleNet.name = 'LittleNet'
+littleNet.net_layout = [64, 64]
+littleNet.conf_to_name()
+
+verryLittleNet = copy.deepcopy(c)
+verryLittleNet.name = 'VerryLittleNet'
+verryLittleNet.net_layout = [64, 32]
+verryLittleNet.conf_to_name()
+
+verryLittleNetDeep = copy.deepcopy(c)
+verryLittleNetDeep.name = 'VerryLittleNetDeep'
+verryLittleNetDeep.net_layout = [64, 32, 32]
+verryLittleNetDeep.conf_to_name()
+
+# configuration = smallNet
+# configuration = smallNetDeep
+# configuration = normalNet
+# configuration = normalSlowDecay
+# configuration = normalSlowLearn
+# configuration = largeNet
+# configuration = deepNet
+# configuration = verryLittleNet
+# configuration = littleNet
+# configuration = verryLittleNetDeep
+# main.run(configuration)
+
+configurations = [smallNet, smallNetDeep, normalNet, normalSlowDecay, normalSlowLearn, largeNet, deepNet, verryLittleNet, littleNet, verryLittleNetDeep]
+
+threads = []
+for conf in configurations:
+    threads.append(threading.Thread(target=main.run, args=conf))
+
+for thread in threads:
+    thread.start()
 
-configs = trange(len(configurations), desc='Benchmarking configs', unit="Configuration")
-for i in configs:
-    main.run(configurations[i])
+for thread in threads:
+    thread.join()
\ No newline at end of file

environment_wrapper.py

@@ -106,9 +106,14 @@ def process_logs(avg_score_history, loss, conf):
     df.to_csv(conf.save_to + conf.name + '/' + conf.name + '.csv')
 
     """ Plot the log history """
+    plt.figure()
     plt.plot([i+1 for i in range(0, len(loss), 2)], loss[::2])
     plt.plot([i+1 for i in range(0, len(avg_score_history), 2)], avg_score_history[::2], '--')
     plt.title(conf.name)
     plt.savefig(conf.save_to + conf.name + '/' + conf.name + '.png', format="png")
     if conf.render:
         plt.show()
+
+def load_logs(file):
+    df = pd.read_csv(file)
+    return df["Score"], df["Average"]
\ No newline at end of file