From 379a3ae75358fa0afac5b54d1a18d4a887250810 Mon Sep 17 00:00:00 2001
From: jvongehr <joel-jerome.vongehr@stud.hs-bochum.de>
Date: Mon, 2 Aug 2021 14:35:42 +0200
Subject: [PATCH] now saves the last batch of data without crashing
---
python/hdf5.py | 34 ++++++++++++++++++++++------------
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/python/hdf5.py b/python/hdf5.py
index 89c4081..af015a8 100644
--- a/python/hdf5.py
+++ b/python/hdf5.py
@@ -1,3 +1,4 @@
+from gensim.utils import chunkize
if __name__ == '__main__':
#%%
import numpy as np
@@ -123,23 +124,32 @@ if __name__ == '__main__':
i += 1
index +=1
- #XTrain.resize(XTrain.shape[0]+trainChunk, axis=0)
- #XTrain[-trainChunk:] = xTrain
+ trainChunk = len(xTrain)
+ if trainChunk != 0:
- #YTrain.resize(YTrain.shape[0]+trainChunk, axis=0)
- #YTrain[-trainChunk:] = yTrain
+ XTrain.resize(XTrain.shape[0]+trainChunk, axis=0)
+ XTrain[-trainChunk:] = xTrain
- #XVal.resize(XVal.shape[0]+valTestChunk, axis=0)
- #XVal[-valTestChunk:] = xVal
+ YTrain.resize(YTrain.shape[0]+trainChunk, axis=0)
+ YTrain[-trainChunk:] = yTrain
- #YVal.resize(YVal.shape[0]+valTestChunk, axis=0)
- #YVal[-valTestChunk:] = yVal
+ valTestChunk = len(xVal)
+ if valTestChunk != 0:
- #XTest.resize(XTest.shape[0]+valTestChunk, axis=0)
- #XTest[-valTestChunk:] = xTest
+ XVal.resize(XVal.shape[0]+valTestChunk, axis=0)
+ XVal[-valTestChunk:] = xVal
- #YTest.resize(YTest.shape[0]+valTestChunk, axis=0)
- #YTest[-valTestChunk:] = yTest
+ YVal.resize(YVal.shape[0]+valTestChunk, axis=0)
+ YVal[-valTestChunk:] = yVal
+
+ valTestChunk = len(xTest)
+ if valTestChunk != 0:
+
+ XTest.resize(XTest.shape[0]+valTestChunk, axis=0)
+ XTest[-valTestChunk:] = xTest
+
+ YTest.resize(YTest.shape[0]+valTestChunk, axis=0)
+ YTest[-valTestChunk:] = yTest
#%%
import h5py
--
GitLab