[SPARK-23081][PYTHON]Add colRegex API to PySpark

Author: huaxingao

python/pyspark/sql/dataframe.py

@@ -1881,6 +1881,15 @@ def toDF(self, *cols):
         jdf = self._jdf.toDF(self._jseq(cols))
         return DataFrame(jdf, self.sql_ctx)
 
+    @since(2.3)
+    def colRegex(self, colName):
+        """
+        Selects column based on the column name specified as a regex and return it
+        as :class:`Column`.
+        """
+        jc = self._jdf.colRegex(colName)
+        return Column(jc)
+
     @since(1.3)
     def toPandas(self):
         """

python/pyspark/sql/tests.py

@@ -2855,6 +2855,10 @@ def test_create_dataframe_from_old_pandas(self):
             with self.assertRaisesRegexp(ImportError, 'Pandas >= .* must be installed'):
                 self.spark.createDataFrame(pdf)
 
+    def test_colRegex(self):
+        df = self.spark.createDataFrame([("a", 1), ("b", 2), ("c",  3)])
+        self.assertEqual(df.select(df.colRegex("`(_1)?+.+`")).collect(), df.select("_2").collect())
+
 
 class HiveSparkSubmitTests(SparkSubmitTests):