Lin*_*ing 0 python numpy hdf5 h5py
我指的是这个问题本。我正在制作这个新主题，是因为我不太了解那里给出的答案，希望有人可以向我进一步解释。
基本上我的问题就像那里的链接一样。之前，我使用它np.vstack并h5从中创建格式文件。以下是我的示例：
import numpy as np
import h5py
import glob
path="/home/ling/test/"
def runtest():
data1 = [np.loadtxt(file) for file in glob.glob(path + "data1/*.csv")]
data2 = [np.loadtxt(file) for file in glob.glob(path + "data2/*.csv")]
stack = np.vstack((data1, data2))
h5f = h5py.File("/home/ling/test/2test.h5", "w")
h5f.create_dataset("test_data", data=stack)
h5f.close()
如果大小都相同，这将非常有效。但是当大小不同时，会抛出错误TypeError: Object dtype dtype('O') has no native HDF5 equivalent
从那里给出的答案可以理解，我必须将数组另存为单独的数据集，但要查看给出的示例代码段；for k,v in adict.items()和grp.create_dataset(k,data=v)，k数据集名称正确吗？就像我的例子一样test_data？那是v什么？
以下是它的外观vstack以及stack
[[array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.07812, -0.07812, -0.11719, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([ 0.03906, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.11719, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.15625, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.11719, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.15625, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.11719, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.11719, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.07812, 0. ])
array([ 0.07812, 0.03906, 0.07812, ..., 0.03906, 0.07812, 0. ])
array([ 0.03906, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([-0.07812, -0.07812, -0.07812, ..., -0.07812, -0.11719, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])
array([ 0.07812, 0.07812, 0.07812, ..., 0.07812, 0.07812, 0. ])]
[ array([ 10.9375 , 10.97656, 10.97656, ..., 11.05469, 11.05469, 1. ])
array([ 11.01562, 11.01562, 11.01562, ..., 11.09375, 11.09375, 1. ])
array([ 11.09375, 11.09375, 11.09375, ..., 11.09375, 11.09375, 1. ])
array([ 10.97656, 11.01562, 11.01562, ..., 11.13281, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.01562, ..., 11.09375, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.05469, 11.13281, 1. ])
array([ 11.05469, 11.09375, 11.09375, ..., 11.09375, 11.09375, 1. ])
array([ 11.09375, 11.05469, 11.09375, ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.09375, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.09375, ..., 11.05469, 11.05469, 1. ])
array([ 10.97656, 10.97656, 10.97656, ..., 11.05469, 11.05469, 1. ])
array([ 11.09375, 11.05469, 11.09375, ..., 11.09375, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.09375, 11.17188, 1. ])
array([ 11.09375, 11.09375, 11.09375, ..., 10.97656, 11.09375, 1. ])
array([ 11.09375, 11.09375, 11.09375, ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.01562, 11.05469, ..., 11.01562, 11.01562, 1. ])
array([ 10.78125, 10.78125, 10.78125, ..., 11.05469, 11.05469, 1. ])
array([ 11.13281, 11.09375, 11.13281, ..., 11.09375, 11.09375, 1. ])
array([ 11.13281, 11.09375, 11.09375, ..., 11.05469, 11.05469, 1. ])
array([ 10.97656, 10.97656, 10.9375 , ..., 11.05469, 11.05469, 1. ])
array([ 11.05469, 11.09375, 11.05469, ..., 11.09375, 11.09375, 1. ])
array([ 10.9375 , 10.9375 , 10.9375 , ..., 11.09375, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.05469, 11.05469, 1. ])
array([ 10.9375 , 10.89844, 10.9375 , ..., 11.05469, 11.09375, 1. ])
array([ 10.9375 , 10.97656, 10.97656, ..., 11.05469, 11.05469, 1. ])
array([ 10.89844, 10.89844, 10.89844, ..., 11.05469, 11.09375, 1. ])
array([ 11.05469, 11.05469, 11.05469, ..., 11.01562, 11.01562, 1. ])]]
感谢您的帮助和解释。
我通过使用熊猫解决了这个问题。最初，我使用了Pierre de Buyl的确切建议，但是当我尝试加载/读取文件/数据集时，它给了我错误。我尝试了test_data = h5f["data1/file1"][:]。这给了我一个错误的说法Unable to open object(Object 'file1' does not exist)。
我通过阅读2test.h5使用进行检查pandas.read_hdf，它显示文件为空。我在网上搜索其他解决方案，发现了这一点。我已经修改了它：
import numpy as np
import glob
import pandas as pd
path = "/home/ling/test/"
def runtest():
data1 = [np.loadtxt(file) for file in glob.glob(path + "data1/*.csv")]
data2 = [np.loadtxt(file) for file in glob.glob(path + "data2/*.csv")]
df1 = pd.DataFrame(data1)
df2 = pd.DataFrame(data2)
combine = df1.append(df2, ignore_index=True)
# sort the NaN to the left
combinedf = combine.apply(lambda x : sorted(x, key=pd.notnull), 1)
combinedf.to_hdf('/home/ling/test/2test.h5', 'twodata')
runtest()
为了阅读，我只是使用
input_data = pd.read_hdf('2test.h5', 'twodata')
read_input = input_data.values
read1 = read_input[:, -1] # read/get last column for example
HDF5文件中的基本元素是组（类似于目录）和数据集（类似于数组）。
NumPy将创建一个包含许多不同输入的数组。当尝试从完全不同的元素（即不同长度）创建数组时，NumPy返回类型为'O'的数组。object_在NumPy参考指南中查找。然后，使用NumPy几乎没有优势，因为它类似于标准的Python列表。
HDF5无法存储类型为“ O”的数组，因为它没有通用数据类型（仅对C结构类型对象提供某些支持）。
解决问题的最明显方法是将数据存储在HDF5数据集中，每个表“一个数据集”。您保留了将数据收集到单个文件中的优势，并且可以对元素进行“类似于字典的访问”。
尝试以下代码：
import numpy as np
import h5py
import glob
path="/home/ling/test/"
def runtest():
h5f = h5py.File("/home/ling/test/2test.h5", "w")
h5f.create_group('data1')
h5f.create_group('data2')
[h5f.create_dataset(file[:-4], data=np.loadtxt(file)) for file in glob.glob(path + "data1/*.csv")]
[h5f.create_dataset(file[:-4], data=np.loadtxt(file)) for file in glob.glob(path + "data2/*.csv")]
h5f.close()
阅读：
h5f = h5py.File("/home/ling/test/2test.h5", "r")
test_data = h5f['data1/thefirstfilenamewithoutcsvextension'][:]