Python-сообщество

dset = f.create_dataset("some_path_in_file", compression=arhopt, 
                shape=(l, N), maxshape=(None, N))
....
dset.resize((new_size, N))

np.mean(fp, axis=0)

fp = np.memmap('C:/data_1000000x1000', dtype='float32', mode='w+', shape=(1000000,1000))

import numpy as np
import time
rows= 10000 # it can be large for example 1kk
cols= 1000
#create some data in memory
data = np.arange(rows*cols, dtype='float32') 
data.resize((rows,cols))
#create file on disk
fp0 = np.memmap('C:/data_0', dtype='float32', mode='w+', shape=(rows,cols))
fp1 = np.memmap('C:/data_1', dtype='float32', mode='w+', shape=(rows,cols))
fp0[:]=data[:]
fp1[:]=data[:]
#matrix transpose test
tr = np.memmap('C:/data_tr', dtype='float32', mode='w+', shape=(cols,rows))
tr= np.transpose(fp1)  #memory consumption?
print fp1.shape
print tr.shape
res = np.memmap('C:/data_res', dtype='float32', mode='w+', shape=(rows,rows))
t0 = time.time()
res[:]= np.dot(fp0,tr) #takes 342 seconds on my machine, if I multiplicate matrices in RAM it takes 345 seconds (I thinks it's a strange result)
print res.shape
print (time.time() - t0)

import numpy as np
import time
import tables
import cProfile
rows = 10000
cols = 1000
batches = 100
k= 10
#USING HDF5
vec= np.random.rand(1,cols)
data = np.random.rand(rows,cols)
fileName = 'C:\carray1.h5'
shape = (rows*batches, cols)  # predefined size
atom = tables.UInt8Atom()  #?
filters = tables.Filters(complevel=5, complib='zlib') #?
# h5f = tables.open_file(fileName, 'w')
# ca = h5f.create_carray(h5f.root, 'carray', atom, shape, filters=filters)
# for i in range(batches):
	# ca[i*rows:(i+1)*rows]= data[:]+i  # +i to modify data
# h5f.close()
def test_knn():
	h5f = tables.open_file(fileName)
	t0= time.time()
	d = np.empty((rows*batches,))
	for i in range(batches):
		d[i*rows:(i+1)*rows] = ((h5f.root.carray[i*rows:(i+1)*rows]-vec)**2).sum(axis=1)
	print (time.time()-t0)
	ndx = d.argsort()
	print ndx[:k]
	h5f.close()
cProfile.run('test_knn()')

def test_tree_knn():
	h5f = tables.open_file(fileName)
        //так работать не будет
	# t0= time.time()
	# nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(h5f.root.carray)
	# distances, indices = nbrs.kneighbors(vec)
	# print (time.time()-t0)
		
        //надо еще distances, indices слеплять на каждой итерации
	t0= time.time()
	d = np.empty((rows*batches,))
	for i in range(batches):
		nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(h5f.root.carray[i*rows:(i+1)*rows])
		distances, indices = nbrs.kneighbors(vec)  # put in dict? 
		#d[i*rows:(i+1)*rows] = 
	print (time.time()-t0)
	#ndx = d.argsort()
	#print ndx[:k]
	h5f.close()