小编BHA*_*H M的帖子

问题陈述:我需要传递一个字符串列表并在spark java中使用filter函数应用like运算符.

原因: 像运算符一样考虑单个字符串实体,因此代码中存在很多性能问题,因为我们需要应用于更大的数据集,为了克服这个问题,我需要传递字符串列表并使用过滤器函数应用类似运算符.

所以请让我知道如何使用类似的应用列表.因为我需要找到类似ID的模式

 JavaSparkContext sc = new JavaSparkContext(new SparkConf().setAppName("SparkJdbcDs").setMaster("local[*]"));
  SQLContext sqlContext = new SQLContext(sc);
  SparkSession spark = SparkSession.builder().appName("JavaTokenizerExample").getOrCreate();

  List<Row> data = Arrays.asList(
      RowFactory.create("J40504", "CRC Industries"),
      RowFactory.create("K630-0746777","Dixon value"),
      RowFactory.create("K444-4444","3M INdustries"),
      RowFactory.create("4333444","3M INdustries"),
      RowFactory.create("566-655","3M INdustries"),
      RowFactory.create("4444888","3M INdustries"),
      RowFactory.create("P477-7444","3M INdustries"),
      RowFactory.create("566655","Dixon coupling valve"));
   // In real time we have large dataset

  StructType schema = new StructType(new StructField[] {new StructField("label1", DataTypes.StringType, false,Metadata.empty()),
    new StructField("sentence1", DataTypes.StringType, false,Metadata.empty()) });

  Dataset<Row> sentenceDataFrame = spark.createDataFrame(data, schema);

  List<String> listStrings = new ArrayList<String>();
  listStrings.add("40504");
  listStrings.add("630-0746");
  listStrings.add("477-7444");
  listStrings.add("444-4444"); …