chore: sync changes from internal repo (#10)

feat: support `Series.corr`
fix: raise AttributeError for unimplemented pandas methods
feat: support `DataFrame.stack`
feat: support `np.arcsin`, `np.arccos`, `np.arctan`, `np.sinh`, `np.cosh`, `np.tanh`, `np.arcsinh`, `np.arccosh`, `np.arctanh`, `np.exp` with Series argument
fix: align column names with pandas in `DataFrame.agg` results
docs: set `options.bigquery.project` in sample code
chore: unit test internal `get_standardized_ids` method
fix: include survey link in abstract `NotImplementedError` exception messages
perf: lazily instantiate client library objects
fix: allow (but still not recommended) `ORDER BY` in `read_gbq` input when an `index_col` is defined
feat: support `read_json` with `engine=bigquery` for newline-delimited JSON files
chore: remove unneeded `types-retry` reference
feat: support `np.sin`, `np.cos`, `np.tan`, `np.log`, `np.log10`, `np.sqrt`, `np.abs` with Series argument
fix: label temp table creation jobs with `source=bigquery-dataframes-temp` label
fix: support spaces in column names in `DataFrame` initializater
chore: fix permissions on publish docs script
feat: support `df[my_column] = [a python list]`
feat: add `components_`, `explained_variance_`, and `explained_variance_ratio_` properties to `bigframes.ml.decomposition.PCA`
chore: add execute permissions on publish docs script
docs: fix link to GitHub
chore: fix docs build
fix: check for IAM role on the BigQuery connection when initializing a `remote_function`
chore: revert pin to maximum pytest-retry plugin version in tests

Author: tswast

.kokoro/docs/common.cfg

@@ -20,7 +20,7 @@ env_vars: {
 }
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "git/bigframes/.kokoro/publish-docs.sh"
+    value: ".kokoro/publish-docs.sh"
 }
 
 env_vars: {

.kokoro/publish-docs.sh

@@ -41,6 +41,7 @@ method accepts either a fully-qualified table ID or a SQL query.
 
   import bigframes.pandas as bpd
 
+  bpd.options.bigquery.project = your_gcp_project_id
   df1 = bpd.read_gbq("project.dataset.table")
   df2 = bpd.read_gbq("SELECT a, b, c, FROM `project.dataset.table`")
 
@@ -260,7 +261,7 @@ To view and manage Cloud Functions functions, use the
 `Functions <https://console.cloud.google.com/functions/list?env=gen2>`_
 page and use the project picker to select the project in which you
 created the function. For easy identification, the names of the functions
-created by BigQuery DataFrames are prefixed by ``bigframes-``.
+created by BigQuery DataFrames are prefixed by ``bigframes``.
 
 **Requirements**
 
@@ -283,7 +284,9 @@ following IAM roles:
 * BigQuery Data Editor (roles/bigquery.dataEditor)
 * BigQuery Connection Admin (roles/bigquery.connectionAdmin)
 * Cloud Functions Developer (roles/cloudfunctions.developer)
-* Service Account User (roles/iam.serviceAccountUser)
+* Service Account User (roles/iam.serviceAccountUser) on the
+  `service account <https://cloud.google.com/functions/docs/reference/iam/roles#additional-configuration> `
+  ``PROJECT_NUMBER-compute@developer.gserviceaccount.com``
 * Storage Object Viewer (roles/storage.objectViewer)
 * Project IAM Admin (roles/resourcemanager.projectIamAdmin)
 
@@ -330,7 +333,7 @@ Data processing location
 
 BigQuery DataFrames is designed for scale, which it achieves by keeping data
 and processing on the BigQuery service. However, you can bring data into the
-memory of your client machine by calling ``.execute()`` on a DataFrame or Series
+memory of your client machine by calling ``.to_pandas()`` on a DataFrame or Series
 object. If you choose to do this, the memory limitation of your client machine
 applies.
 

README.rst

@@ -21,3 +21,5 @@
     "Share your usecase with the BigQuery DataFrames team at the "
     "https://bit.ly/bigframes-feedback survey."
 )
+
+ABSTRACT_METHOD_ERROR_MESSAGE = f"Abstract method. You have likely encountered a bug. Please share this stacktrace and how you reached it with the BigQuery DataFrames team. {FEEDBACK_LINK}"

bigframes/constants.py

@@ -35,6 +35,7 @@
     reencode_order_string,
     StringEncoding,
 )
+import bigframes.core.utils as utils
 import bigframes.dtypes
 import bigframes.operations as ops
 import bigframes.operations.aggregations as agg_ops
@@ -562,6 +563,36 @@ def aggregate(
                 ordering=ordering,
             )
 
+    def corr_aggregate(
+        self, corr_aggregations: typing.Sequence[typing.Tuple[str, str, str]]
+    ) -> ArrayValue:
+        """
+        Get correlations between each lef_column_id and right_column_id, stored in the respective output_column_id.
+        This uses BigQuery's CORR under the hood, and thus only Pearson's method is used.
+        Arguments:
+            corr_aggregations: left_column_id, right_column_id, output_column_id tuples
+        """
+        table = self.to_ibis_expr(ordering_mode="unordered")
+        stats = {
+            col_out: table[col_left].corr(table[col_right], how="pop")
+            for col_left, col_right, col_out in corr_aggregations
+        }
+        aggregates = {**stats, ORDER_ID_COLUMN: ibis_types.literal(0)}
+        result = table.aggregate(**aggregates)
+        # Ordering is irrelevant for single-row output, but set ordering id regardless as other ops(join etc.) expect it.
+        ordering = ExpressionOrdering(
+            ordering_value_columns=[OrderingColumnReference(ORDER_ID_COLUMN)],
+            total_ordering_columns=frozenset([ORDER_ID_COLUMN]),
+            integer_encoding=IntegerEncoding(is_encoded=True, is_sequential=True),
+        )
+        return ArrayValue(
+            self._session,
+            result,
+            columns=[result[col_id] for col_id in [*stats.keys()]],
+            hidden_ordering_columns=[result[ORDER_ID_COLUMN]],
+            ordering=ordering,
+        )
+
     def project_window_op(
         self,
         column_name: str,
@@ -852,52 +883,91 @@ def _ibis_window_from_spec(self, window_spec: WindowSpec, allow_ties: bool = Fal
             group_by=group_by,
         )
 
-    def unpivot_single_row(
+    def unpivot(
         self,
         row_labels: typing.Sequence[typing.Hashable],
-        unpivot_columns: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]],
+        unpivot_columns: typing.Sequence[
+            typing.Tuple[str, typing.Sequence[typing.Optional[str]]]
+        ],
         *,
+        passthrough_columns: typing.Sequence[str] = (),
         index_col_id: str = "index",
-        dtype=pandas.Float64Dtype(),
+        dtype: typing.Union[
+            bigframes.dtypes.Dtype, typing.Sequence[bigframes.dtypes.Dtype]
+        ] = pandas.Float64Dtype(),
     ) -> ArrayValue:
-        """Unpivot a single row."""
-        # TODO: Generalize to multiple row input
-        table = self.to_ibis_expr(ordering_mode="unordered")
+        """
+        Unpivot ArrayValue columns.
+
+        Args:
+            row_labels: Identifies the source of the row. Must be equal to length to source column list in unpivot_columns argument.
+            unpivot_columns: Mapping of column id to list of input column ids. Lists of input columns may use None.
+            passthrough_columns: Columns that will not be unpivoted. Column id will be preserved.
+            index_col_id (str): The column id to be used for the row labels.
+            dtype (dtype or list of dtype): Dtype to use for the unpivot columns. If list, must be equal in number to unpivot_columns.
+
+        Returns:
+            ArrayValue: The unpivoted ArrayValue
+        """
+        table = self.to_ibis_expr(ordering_mode="offset_col")
         sub_expressions = []
 
-        # TODO: validate all columns are equal length, as well as row labels
+        # Use ibis memtable to infer type of rowlabels (if possible)
+        # TODO: Allow caller to specify dtype
+        labels_ibis_type = ibis.memtable({"col": row_labels})["col"].type()
+        labels_dtype = bigframes.dtypes.ibis_dtype_to_bigframes_dtype(labels_ibis_type)
+
         row_n = len(row_labels)
         if not all(
             len(source_columns) == row_n for _, source_columns in unpivot_columns
         ):
             raise ValueError("Columns and row labels must all be same length.")
 
-        # Select each column
         for i in range(row_n):
             values = []
-            for result_col, source_cols in unpivot_columns:
-                values.append(
-                    ops.AsTypeOp(dtype)._as_ibis(table[source_cols[i]]).name(result_col)
-                )
-
+            for j in range(len(unpivot_columns)):
+                result_col, source_cols = unpivot_columns[j]
+                col_dtype = dtype[j] if utils.is_list_like(dtype) else dtype
+                if source_cols[i] is not None:
+                    values.append(
+                        ops.AsTypeOp(col_dtype)
+                        ._as_ibis(table[source_cols[i]])
+                        .name(result_col)
+                    )
+                else:
+                    values.append(
+                        bigframes.dtypes.literal_to_ibis_scalar(
+                            None, force_dtype=col_dtype
+                        ).name(result_col)
+                    )
+            offsets_value = (
+                ((table[ORDER_ID_COLUMN] * row_n) + i)
+                .cast(ibis_dtypes.int64)
+                .name(ORDER_ID_COLUMN),
+            )
             sub_expr = table.select(
-                ibis_types.literal(row_labels[i]).name(index_col_id),
+                passthrough_columns,
+                bigframes.dtypes.literal_to_ibis_scalar(
+                    row_labels[i], force_dtype=labels_dtype  # type:ignore
+                ).name(index_col_id),
                 *values,
-                ibis_types.literal(i).name(ORDER_ID_COLUMN),
+                offsets_value,
             )
             sub_expressions.append(sub_expr)
         rotated_table = ibis.union(*sub_expressions)
 
         value_columns = [
             rotated_table[value_col_id] for value_col_id, _ in unpivot_columns
         ]
+        passthrough_values = [rotated_table[col] for col in passthrough_columns]
         return ArrayValue(
             session=self._session,
             table=rotated_table,
-            columns=[rotated_table[index_col_id], *value_columns],
+            columns=[rotated_table[index_col_id], *value_columns, *passthrough_values],
             hidden_ordering_columns=[rotated_table[ORDER_ID_COLUMN]],
             ordering=ExpressionOrdering(
                 ordering_value_columns=[OrderingColumnReference(ORDER_ID_COLUMN)],
+                integer_encoding=IntegerEncoding(is_encoded=True, is_sequential=True),
                 total_ordering_columns=frozenset([ORDER_ID_COLUMN]),
             ),
         )

bigframes/core/__init__.py

@@ -197,3 +197,35 @@ def rank(
             )
 
     return block.select_columns(rownum_col_ids).with_column_labels(labels)
+
+
+def dropna(block: blocks.Block, how: typing.Literal["all", "any"] = "any"):
+    """
+    Drop na entries from block
+    """
+    if how == "any":
+        filtered_block = block
+        for column in block.value_columns:
+            filtered_block, result_id = filtered_block.apply_unary_op(
+                column, ops.notnull_op
+            )
+            filtered_block = filtered_block.filter(result_id)
+            filtered_block = filtered_block.drop_columns([result_id])
+        return filtered_block
+    else:  # "all"
+        filtered_block = block
+        predicate = None
+        for column in block.value_columns:
+            filtered_block, partial_predicate = filtered_block.apply_unary_op(
+                column, ops.notnull_op
+            )
+            if predicate:
+                filtered_block, predicate = filtered_block.apply_binary_op(
+                    partial_predicate, predicate, ops.or_op
+                )
+            else:
+                predicate = partial_predicate
+        if predicate:
+            filtered_block = filtered_block.filter(predicate)
+        filtered_block = filtered_block.select_columns(block.value_columns)
+        return filtered_block