diff --git a/python/hdf5.py b/python/hdf5.py index 89c4081c63a1c8d252fc00ed46830d6c888129b4..af015a81eeefd4412df8f5a1f2c3cbfb2ad168b5 100644 --- a/python/hdf5.py +++ b/python/hdf5.py @@ -1,3 +1,4 @@ +from gensim.utils import chunkize if __name__ == '__main__': #%% import numpy as np @@ -123,23 +124,32 @@ if __name__ == '__main__': i += 1 index +=1 - #XTrain.resize(XTrain.shape[0]+trainChunk, axis=0) - #XTrain[-trainChunk:] = xTrain + trainChunk = len(xTrain) + if trainChunk != 0: - #YTrain.resize(YTrain.shape[0]+trainChunk, axis=0) - #YTrain[-trainChunk:] = yTrain + XTrain.resize(XTrain.shape[0]+trainChunk, axis=0) + XTrain[-trainChunk:] = xTrain - #XVal.resize(XVal.shape[0]+valTestChunk, axis=0) - #XVal[-valTestChunk:] = xVal + YTrain.resize(YTrain.shape[0]+trainChunk, axis=0) + YTrain[-trainChunk:] = yTrain - #YVal.resize(YVal.shape[0]+valTestChunk, axis=0) - #YVal[-valTestChunk:] = yVal + valTestChunk = len(xVal) + if valTestChunk != 0: - #XTest.resize(XTest.shape[0]+valTestChunk, axis=0) - #XTest[-valTestChunk:] = xTest + XVal.resize(XVal.shape[0]+valTestChunk, axis=0) + XVal[-valTestChunk:] = xVal - #YTest.resize(YTest.shape[0]+valTestChunk, axis=0) - #YTest[-valTestChunk:] = yTest + YVal.resize(YVal.shape[0]+valTestChunk, axis=0) + YVal[-valTestChunk:] = yVal + + valTestChunk = len(xTest) + if valTestChunk != 0: + + XTest.resize(XTest.shape[0]+valTestChunk, axis=0) + XTest[-valTestChunk:] = xTest + + YTest.resize(YTest.shape[0]+valTestChunk, axis=0) + YTest[-valTestChunk:] = yTest #%% import h5py