在 Tensorflow 中过滤数据

Question

我在 TensorFlow 中有一些列数据，我想过滤其中一列，如下所示：

import pandas as pd

import tensorflow.compat.v2 as tf
import tensorflow.compat.v1 as tfv1
tfv1.enable_v2_behavior()

csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')

df = pd.read_csv(csv_file)
target = df.pop('target')
df['thal'] = pd.Categorical(df['thal'])
df['thal'] = df.thal.cat.codes

# Use interleave() and prefetch() to read many files concurrently.
#files = tf.data.Dataset.list_files(file_pattern=input_file_pattern, shuffle=True, seed=123456789)
#dataset = files.interleave(lambda x: tf.data.RecordIODataset(x).prefetch(100), cycle_length=8)

#Pretend I actually had some data files
dataset = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values))

dataset = dataset.shuffle(1000, seed=123456789)
dataset = dataset.batch(20)
#Pretend I did some parsing here
# dataset = dataset.map(parse_record, num_parallel_calls=20) 
dataset = dataset.filter(lambda x, label: x['trestbps']<135)

但这会产生错误消息：

ValueError：predicate返回类型必须可转换为标量布尔张量。曾是TensorSpec(shape=(None,), dtype=tf.bool, name=None)。

我应该做什么来过滤数据？

Answer 1

这是因为您filter在 后应用了batch. 因此，在lambda表达式中，x是具有形状的批次(None,)（传递drop_reminder=True到batch以获得 的形状(20,)），而不是样本。要修复它，您必须filter先调用batch.

有一个解决方案可以在 后“过滤” batch，使用 amap代替。然而，正如您所看到的，这对批量处理变量大小有一个副作用：您在输入中获得了 20 个批次，并且删除了不符合特定条件 (trestbps < 135) 的元素，而不是删除了相同数量的元素每一批。而且这个解决方案的表现非常糟糕......

import timeit

import pandas as pd

import tensorflow.compat.v2 as tf
import tensorflow.compat.v1 as tfv1
tfv1.enable_v2_behavior()

def s1(ds):
    dataset = ds
    dataset = dataset.filter(lambda x, label: x['trestbps']<135)
    dataset = dataset.batch(20)
    return dataset

def s2(ds):
    dataset = ds
    dataset = dataset.batch(20)
    dataset = dataset.map(lambda x, label: (tf.nest.map_structure(lambda y: y[x['trestbps'] < 135], x), label[x['trestbps'] < 135]))
    return dataset


def base_ds():
    csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')

    df = pd.read_csv(csv_file)
    target = df.pop('target')
    df['thal'] = pd.Categorical(df['thal'])
    df['thal'] = df.thal.cat.codes

    return tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values))


def main():
    ds = base_ds()
    ds1 = s1(ds)
    ds2 = s2(ds)
    tf.print("DS_S1:", [tf.nest.map_structure(lambda x: x.shape, x) for x in ds1])
    tf.print("DS_S2:", [tf.nest.map_structure(lambda x: x.shape, x) for x in ds2])
    tf.print("Are equals?", [x for x in ds1] == [x for x in ds2])
    tf.print("Contains same elements?", [x for x in ds1.unbatch()] == [x for x in ds2.unbatch()])

    tf.print("Filter and batch:", timeit.timeit(lambda: s1(ds), number=100))
    tf.print("Batch and map:", timeit.timeit(lambda: s2(ds), number=100))

if __name__ == '__main__':
    main()

结果是：

# Tensor shapes
[...]
Are equals? False
Contains same elements? True
Filter and batch: 0.5571189750007761
Batch and map: 15.582061060000342

种类