diff --git a/prompt.txt b/prompt.txt index 9de97f82..8861882f 100644 --- a/prompt.txt +++ b/prompt.txt @@ -107,21 +107,28 @@ class SynSeqPreprocessor: - Records the original dtypes. - Automatically assigns dtypes (date/category/numeric) when not provided. - Converts date columns to datetime and category columns to 'category' dtype. - - For numeric columns, if one value accounts for ≥90% of non-null rows, that value is + - For numeric columns, if one value accounts for ≥90% of non‐null rows, that value is automatically marked as a special value. For each such column, a new categorical column (named base_col_cat) is created: - * If the cell value is missing, it is mapped to "NAN". - * If the cell value equals a detected (or user-specified) special value, it is mapped to that special value (as string). - * Otherwise, it is marked as "NUMERIC". + * If the cell value is missing, it is mapped to the missing marker (here, -99999999). + * If the cell value equals a detected (or user‐specified) special value, it is left as its + original numeric value. + * Otherwise, it is marked with the “numeric” marker (here, -999999999) indicating that the + value is not special. Postprocessing: - Merges back the split (base_col, base_col_cat) columns: For rows where the base column is NaN and the corresponding _cat column - indicates a special value (i.e. not "NUMERIC"), the base column is replaced - with that special value. In particular, if _cat equals "NAN", the base column is set to np.nan. - - Optionally applies user-provided rules sequentially to filter rows. + is not equal to the numeric marker, the base column is replaced + with that special value. In particular, if _cat equals the missing marker, + the base column is set to np.nan. + - Optionally applies user‐provided rules sequentially to filter rows. """ + # Define marker constants (choose values unlikely to appear in your data) + NUMERIC_MARKER = -777777777.0 # Indicates a normal (non‐special) value + MISSING_MARKER = -999999999.0 # Indicates a missing value + def __init__( self, user_dtypes: Optional[Dict[str, str]] = None, @@ -131,7 +138,7 @@ class SynSeqPreprocessor: """ Args: user_dtypes: {col: "date"/"category"/"numeric"}, if not provided, auto-detected. - user_special_values: {col: [special_value1, special_value2, ...]}. + user_special_values: {col: [special_value1, special_value2, ...]}. Even if not provided, special values are detected automatically for imbalanced numeric columns. max_categories: When auto-detecting dtypes, if nunique <= max_categories, assign 'category', else 'numeric'. """ @@ -140,8 +147,8 @@ class SynSeqPreprocessor: self.max_categories = max_categories # Internal storage - self.original_dtypes: Dict[str, str] = {} # {col: original_dtype} - self.split_map: Dict[str, str] = {} # {base_col -> cat_col} + self.original_dtypes: Dict[str, str] = {} # {col: original_dtype} + self.split_map: Dict[str, str] = {} # {base_col -> cat_col} self.detected_specials: Dict[str, List[Any]] = {} # stores the special values (detected or user-provided) # ========================================================================= @@ -249,11 +256,11 @@ class SynSeqPreprocessor: def _split_numeric_columns(self, df: pd.DataFrame): """ For each numeric column in user_special_values: - - Create a new categorical column (named base_col_cat) that marks special values. + - Create a new numeric column (named base_col_cat) that marks special values using integer markers. - For each cell in the base column: - If NaN -> returns "NAN". - If the value is in the list of special values -> returns that special value (as string). - Otherwise -> returns "NUMERIC". + If NaN -> returns the missing marker (-99999999). + If the value is in the list of special values -> returns that special value. + Otherwise -> returns the numeric marker (-999999999). """ for col, specials in self.user_special_values.items(): if col not in df.columns: @@ -261,24 +268,22 @@ class SynSeqPreprocessor: cat_col = col + "_cat" self.split_map[col] = cat_col - # Store the complete list of special values (detected or user provided) self.detected_specials[col] = specials - # Remove any existing cat_col. if cat_col in df.columns: df.drop(columns=[cat_col], inplace=True) base_idx = df.columns.get_loc(col) df.insert(base_idx, cat_col, None) - def cat_mapper(x, specials, normal_marker="NUMERIC", missing_marker="NAN"): + def cat_mapper(x, specials, normal_marker=self.NUMERIC_MARKER, missing_marker=self.MISSING_MARKER): if pd.isna(x): - return missing_marker + return float(missing_marker) elif x in specials: - return str(x) + return float(x) # retain the special value as-is else: - return normal_marker - df[cat_col] = df[col].apply(lambda x: cat_mapper(x, specials)).astype(str) - df[cat_col] = df[cat_col].astype("category") + return float(normal_marker) + + df[cat_col] = df[col].apply(lambda x: cat_mapper(x, specials)) # ========================================================================= # POSTPROCESSING @@ -288,7 +293,7 @@ class SynSeqPreprocessor: Postprocesses the synthetic DataFrame: 1) Merges back split columns (base_col, base_col_cat) by replacing NaNs in the base column with the corresponding special value (if _cat indicates a special value). - In particular, if _cat equals "NAN", the base column is set to np.nan. + In particular, if _cat equals the missing marker, the base column is set to np.nan. 2) Optionally applies user-provided rules sequentially to filter rows. (Note: Date offset restoration is not performed.) """ @@ -303,17 +308,16 @@ class SynSeqPreprocessor: def _merge_splitted_cols(self, df: pd.DataFrame) -> pd.DataFrame: """ For each (base_col, cat_col) pair in split_map: - - If a base column cell is NaN and the corresponding _cat cell is not "NUMERIC", + - If a base column cell is NaN and the corresponding _cat cell is not equal to the numeric marker, then replace the base column cell. - * If the _cat cell is "NAN", set the base column cell to np.nan. - * Otherwise, convert the _cat cell back to its original special value. + * If the _cat cell equals the missing marker, set the base column cell to np.nan. + * Otherwise, set the base column cell to the value in the _cat cell. - Finally, drop the auxiliary _cat column. """ for base_col, cat_col in self.split_map.items(): if base_col in df.columns and cat_col in df.columns: specials = self.detected_specials.get(base_col, []) - # Condition: base column is NaN and _cat is not "NUMERIC" - condition = df[base_col].isna() & (~df[cat_col].isin(["NUMERIC"])) + condition = df[base_col].isna() & (df[cat_col] != self.NUMERIC_MARKER) if condition.any(): df.loc[condition, base_col] = df.loc[condition, cat_col].apply( lambda v: self._convert_special_value(v, specials) @@ -321,16 +325,15 @@ class SynSeqPreprocessor: df.drop(columns=[cat_col], inplace=True) return df - def _convert_special_value(self, val: str, specials: List[Any]) -> Any: + def _convert_special_value(self, val: Any, specials: List[Any]) -> Any: """ - Given the string representation of a special value and the list of original special values, - returns the original special value. In particular, if val equals "NAN", return np.nan. + Given the numeric marker from a _cat column and the list of original special values, + returns the original special value. In particular, if val equals the missing marker, + returns np.nan. """ - if val == "NAN": + if val == self.MISSING_MARKER: return np.nan - for special in specials: - if str(special) == val: - return special + # If the value is one of the special values, return it; otherwise, return the value as-is. return val def apply_rules(self, df: pd.DataFrame, rules: Dict[str, List[Tuple[str, str, Any]]]) -> pd.DataFrame: @@ -397,9 +400,9 @@ This module implements a sequential synthesizer (Syn_Seq) that fits each column of the data (following an ordering provided in syn_order) one by one. It supports columns with special values. For such columns, during fitting the model only sees rows with numeric (non‐special) values. At generation time, the -pre‐generated categorical (“_cat”) column (injected by the preprocessor) is used -to decide which rows should be generated by the fitted model (when _cat equals "NUMERIC") -and which rows should directly receive the special value. +pre‐generated special indicator (“_cat”) column (injected by the preprocessor) is +used to decide which rows should be generated by the fitted model (when the _cat +cell equals the numeric marker) and which rows should directly receive the special value. """ from typing import Any, Dict, List, Optional, Tuple @@ -435,6 +438,9 @@ METHOD_MAP: Dict[str, Tuple[Any, Any]] = { "swr": (syn_swr, generate_swr), } +# --- Marker constants (changeable if needed) --- +DEFAULT_NUMERIC_MARKER = -777777777.0 # Indicates that the value is a “normal” (numeric) value +DEFAULT_MISSING_MARKER = -999999999.0 # Indicates that the original cell was missing class Syn_Seq: def __init__( @@ -450,8 +456,11 @@ class Syn_Seq: self.random_state = random_state self.sampling_patience = sampling_patience - # special_values: mapping base column name -> list of special values (e.g. {"capital-gain": [0]}) + # special_values: mapping base column name -> list of special values (e.g. { "capital-gain": [0] }) self.special_values: Dict[str, List[Any]] = {} + # cat_distributions: for each column with a _cat indicator, store its distribution from training. + self.cat_distributions: Dict[str, Dict[Any, float]] = {} + self._model_trained = False self._syn_order: List[str] = [] # synthesis order of columns @@ -459,14 +468,14 @@ class Syn_Seq: self._varsel: Dict[str, List[str]] = {} # predictors for synthesizing each column self._col_models: Dict[str, Dict[str, Any]] = {} # fitted model for each column - # For the first column—and for each numeric column with special values— - # store the observed (real) distribution (filtered to numeric values only) - self._stored_col_data: Dict[str, np.ndarray] = {} + # For the first column, store its observed (real) distribution. + self._first_col_distribution: Dict[str, np.ndarray] = {} def fit_col(self, loader: Any, *args: Any, **kwargs: Any) -> "Syn_Seq": """ Fit each column sequentially using metadata from the loader. - Also extracts special values from any _cat columns. + Also extracts special values from any _cat columns and computes the training + distribution of those columns. """ info_dict = loader.info() training_data = loader.dataframe().copy() @@ -478,14 +487,20 @@ class Syn_Seq: self._method_map = info_dict.get("method", {}) self._varsel = info_dict.get("variable_selection", {}) - # --- Extract special values from any _cat columns in the training data --- + # --- Extract special values and _cat column distributions from the training data --- for col in training_data.columns: if col.endswith("_cat"): base_col = col[:-4] - specials = training_data[col].unique() - specials = [v for v in specials if v not in ["NUMERIC"]] + # Extract all unique values except the numeric and missing markers + specials = training_data[col].unique().tolist() + specials = [v for v in specials if v != DEFAULT_NUMERIC_MARKER and v != DEFAULT_MISSING_MARKER] if specials: self.special_values[base_col] = specials + valid = training_data[col].dropna() + if len(valid) > 0: + self.cat_distributions[base_col] = valid.value_counts(normalize=True).to_dict() + else: + self.cat_distributions[base_col] = {DEFAULT_NUMERIC_MARKER: 1.0} # For auto-injected _cat columns, force method "cart" and mirror variable selection. for col in self._syn_order: @@ -500,16 +515,16 @@ class Syn_Seq: print("[INFO] Syn_Seq aggregator: fitting columns...") - # (1) For the first column, store its real (non-null) distribution. + # (1) For the first column, store its observed (non-null) distribution. first_col = self._syn_order[0] - self._stored_col_data[first_col] = training_data[first_col].dropna().values + self._first_col_distribution[first_col] = training_data[first_col].dropna().values - # (2) For columns with special values, store only rows that are NOT special. + # (2) For columns with special values, store only rows that are not special. for col, specials in self.special_values.items(): if col not in training_data.columns: continue filtered = training_data[~training_data[col].isin(specials)] - self._stored_col_data[col] = filtered[col].dropna().values + self._first_col_distribution[col] = filtered[col].dropna().values print(f"Fitting '{first_col}' => stored distribution from real data. Done.") @@ -553,9 +568,12 @@ class Syn_Seq: """ Generate `count` rows sequentially. - For columns with special values, the pre-generated _cat column (e.g. "capital-gain_cat") + For columns with special values, the pre‐generated numeric indicator (“_cat”) column is used to decide which rows should have their value generated using the fitted model - (if the _cat cell equals "NUMERIC") and which rows should directly receive the special value. + (if the _cat cell equals the DEFAULT_NUMERIC_MARKER) and which rows should directly + receive the special value. + If the generated _cat column contains no DEFAULT_NUMERIC_MARKER flag, it is re‐sampled + using the stored training distribution. """ if not self._model_trained: raise RuntimeError("Syn_Seq aggregator not yet fitted") @@ -565,10 +583,13 @@ class Syn_Seq: # Initialize a DataFrame with NaN values for all columns. gen_df = pd.DataFrame({col: [np.nan] * count for col in self._syn_order}) - # (1) Generate the first column using its stored real distribution. + # (1) Generate the first column using its stored distribution. first_col = self._syn_order[0] - if self._stored_col_data.get(first_col) is not None and len(self._stored_col_data[first_col]) > 0: - gen_df[first_col] = np.random.choice(self._stored_col_data[first_col], size=count, replace=True) + if (first_col in self._first_col_distribution and + len(self._first_col_distribution[first_col]) > 0): + gen_df[first_col] = np.random.choice( + self._first_col_distribution[first_col], size=count, replace=True + ) else: gen_df[first_col] = 0 print(f"Generating '{first_col}' => done.") @@ -576,41 +597,44 @@ class Syn_Seq: # (2) For each subsequent column, generate synthetic values. for col in self._syn_order[1:]: method_name = self._method_map.get(col, "cart") - idx = self._syn_order.index(col) - preds_list = self._varsel.get(col, self._syn_order[:idx]) + preds_list = self._varsel.get(col, self._syn_order[:self._syn_order.index(col)]) - if col in self.special_values: - # For columns with special values, rely on the pre-generated _cat column. - cat_col = col + "_cat" - if cat_col not in gen_df.columns: - raise RuntimeError(f"Expected categorical column {cat_col} not found in generated data.") - - # Check the distribution of the categorical column. - numeric_count = (gen_df[cat_col] == "NUMERIC").sum() + # Check if the column has a corresponding _cat indicator. + cat_col = col + "_cat" + if col in self.special_values and cat_col in gen_df.columns: + # Check if the generated _cat column has any numeric marker. + numeric_count = (gen_df[cat_col] == DEFAULT_NUMERIC_MARKER).sum() if numeric_count == 0: warnings.warn( - f"Degenerate _cat column generated for {col}: no rows marked as 'NUMERIC'. " - "Falling back to random sampling from stored non-special values." + f"Degenerate _cat column for {col}: no rows marked as numeric. Re-sampling _cat column using training distribution." ) - if col in self._stored_col_data and len(self._stored_col_data[col]) > 0: - fallback_values = np.random.choice( - self._stored_col_data[col], size=count, replace=True + cat_dist = self.cat_distributions.get(col, {DEFAULT_NUMERIC_MARKER: 1.0}) + total = sum(cat_dist.values()) + cat_probs = {k: v / total for k, v in cat_dist.items()} + new_cat = np.where( + np.random.rand(count) < cat_probs.get(DEFAULT_NUMERIC_MARKER, 1.0), + float(DEFAULT_NUMERIC_MARKER), + np.random.choice( + [k for k in cat_probs if k != DEFAULT_NUMERIC_MARKER], + size=count, + p=[cat_probs[k] for k in cat_probs if k != DEFAULT_NUMERIC_MARKER] ) - gen_df[col] = fallback_values - else: - gen_df[col] = 0.0 - else: - # For rows with _cat equal to "NUMERIC", generate synthetic values. - is_numeric = gen_df[cat_col] == "NUMERIC" - if is_numeric.sum() > 0: - Xsyn_numeric = gen_df.loc[is_numeric, preds_list].values - ysyn_numeric = self._generate_single_col(method_name, Xsyn_numeric, col) - gen_df.loc[is_numeric, col] = ysyn_numeric - # For rows where _cat indicates a special value, assign that value directly. - for special in self.special_values[col]: - is_special = gen_df[cat_col] == str(special) - gen_df.loc[is_special, col] = float(special) + ) + gen_df[cat_col] = new_cat + gen_df[cat_col] = gen_df[cat_col].astype(float) + + # For rows with _cat equal to the numeric marker, generate synthetic values. + is_numeric = gen_df[cat_col] == DEFAULT_NUMERIC_MARKER + if is_numeric.sum() > 0: + Xsyn_numeric = gen_df.loc[is_numeric, preds_list].values + ysyn_numeric = self._generate_single_col(method_name, Xsyn_numeric, col) + gen_df.loc[is_numeric, col] = ysyn_numeric + # For rows where _cat equals a special value, assign that special value directly. + for special in self.special_values[col]: + is_special = gen_df[cat_col] == special + gen_df.loc[is_special, col] = special else: + # Otherwise, generate the column normally. Xsyn = gen_df[preds_list].values ysyn = self._generate_single_col(method_name, Xsyn, col) gen_df[col] = ysyn @@ -620,19 +644,16 @@ class Syn_Seq: def _generate_single_col(self, method_name: str, Xsyn: np.ndarray, col: str) -> np.ndarray: """ Generate synthetic values for a single column using the fitted model. - If no model is available, falls back to sampling from the stored real distribution. + If no model is available for the column, a RuntimeError is raised. """ if col not in self._col_models or self._col_models[col] is None: - if col in self._stored_col_data and len(self._stored_col_data[col]) > 0: - return np.random.choice(self._stored_col_data[col], size=len(Xsyn), replace=True) - else: - return np.zeros(len(Xsyn)) - + raise RuntimeError(f"No model available for column {col}.") fit_info = self._col_models[col] _, generate_func = METHOD_MAP[fit_info["name"]] return generate_func(fit_info["fitted_model"], Xsyn) + src/synthcity/plugins/core/models/syn_seq/syn_seq_rules.py ** Not a valid file path or file does not exist. ** diff --git a/src/synthcity/plugins/core/models/syn_seq/syn_seq.py b/src/synthcity/plugins/core/models/syn_seq/syn_seq.py index c6f601bc..854da161 100644 --- a/src/synthcity/plugins/core/models/syn_seq/syn_seq.py +++ b/src/synthcity/plugins/core/models/syn_seq/syn_seq.py @@ -5,9 +5,9 @@ of the data (following an ordering provided in syn_order) one by one. It supports columns with special values. For such columns, during fitting the model only sees rows with numeric (non‐special) values. At generation time, the -pre‐generated categorical (“_cat”) column (injected by the preprocessor) is used -to decide which rows should be generated by the fitted model (when _cat equals "NUMERIC") -and which rows should directly receive the special value. +pre‐generated special indicator (“_cat”) column (injected by the preprocessor) is +used to decide which rows should be generated by the fitted model (when the _cat +cell equals the numeric marker) and which rows should directly receive the special value. """ from typing import Any, Dict, List, Optional, Tuple @@ -43,6 +43,9 @@ "swr": (syn_swr, generate_swr), } +# --- Marker constants (changeable if needed) --- +DEFAULT_NUMERIC_MARKER = -777777777.0 # Indicates that the value is a “normal” (numeric) value +DEFAULT_MISSING_MARKER = -999999999.0 # Indicates that the original cell was missing class Syn_Seq: def __init__( @@ -58,12 +61,11 @@ def __init__( self.random_state = random_state self.sampling_patience = sampling_patience - # special_values: mapping base column name -> list of special values (e.g. {"capital-gain": [0]}) + # special_values: mapping base column name -> list of special values (e.g. { "capital-gain": [0] }) self.special_values: Dict[str, List[Any]] = {} - # cat_distributions: mapping base column name -> distribution of the _cat column, - # for example: {"capital-gain": {"NUMERIC": 0.7, "0": 0.3}} - self.cat_distributions: Dict[str, Dict[str, float]] = {} - + # cat_distributions: for each column with a _cat indicator, store its distribution from training. + self.cat_distributions: Dict[str, Dict[Any, float]] = {} + self._model_trained = False self._syn_order: List[str] = [] # synthesis order of columns @@ -71,15 +73,14 @@ def __init__( self._varsel: Dict[str, List[str]] = {} # predictors for synthesizing each column self._col_models: Dict[str, Dict[str, Any]] = {} # fitted model for each column - # For the first column—and for each numeric column (or numeric part of a column with special values)— - # store the observed (real) distribution (filtered to numeric values only) + # For the first column, store its observed (real) distribution. self._first_col_distribution: Dict[str, np.ndarray] = {} def fit_col(self, loader: Any, *args: Any, **kwargs: Any) -> "Syn_Seq": """ Fit each column sequentially using metadata from the loader. - Also extracts special values from any _cat columns and computes the distribution - (i.e. relative frequencies) from those columns. + Also extracts special values from any _cat columns and computes the training + distribution of those columns. """ info_dict = loader.info() training_data = loader.dataframe().copy() @@ -91,17 +92,20 @@ def fit_col(self, loader: Any, *args: Any, **kwargs: Any) -> "Syn_Seq": self._method_map = info_dict.get("method", {}) self._varsel = info_dict.get("variable_selection", {}) - # --- Extract _cat column info from the training data --- + # --- Extract special values and _cat column distributions from the training data --- for col in training_data.columns: if col.endswith("_cat"): base_col = col[:-4] - # Compute the distribution from the _cat column. - dist = training_data[col].value_counts(normalize=True).to_dict() - self.cat_distributions[base_col] = dist - # Special values are those different from "NUMERIC" - specials = [k for k in dist.keys() if k != "NUMERIC"] + # Extract all unique values except the numeric and missing markers + specials = training_data[col].unique().tolist() + specials = [v for v in specials if v != DEFAULT_NUMERIC_MARKER and v != DEFAULT_MISSING_MARKER] if specials: self.special_values[base_col] = specials + valid = training_data[col].dropna() + if len(valid) > 0: + self.cat_distributions[base_col] = valid.value_counts(normalize=True).to_dict() + else: + self.cat_distributions[base_col] = {DEFAULT_NUMERIC_MARKER: 1.0} # For auto-injected _cat columns, force method "cart" and mirror variable selection. for col in self._syn_order: @@ -116,11 +120,11 @@ def fit_col(self, loader: Any, *args: Any, **kwargs: Any) -> "Syn_Seq": print("[INFO] Syn_Seq aggregator: fitting columns...") - # (1) For the first column, store its real (non-null) distribution. + # (1) For the first column, store its observed (non-null) distribution. first_col = self._syn_order[0] self._first_col_distribution[first_col] = training_data[first_col].dropna().values - # (2) For columns with special values, store only rows that are NOT special. + # (2) For columns with special values, store only rows that are not special. for col, specials in self.special_values.items(): if col not in training_data.columns: continue @@ -169,11 +173,12 @@ def generate_col(self, count: int) -> pd.DataFrame: """ Generate `count` rows sequentially. - For columns with special values, the pre‐generated categorical (“_cat”) column (e.g. "capital-gain_cat") + For columns with special values, the pre‐generated numeric indicator (“_cat”) column is used to decide which rows should have their value generated using the fitted model - (if the _cat cell equals "NUMERIC") and which rows should directly receive the special value. - If the generated _cat column contains no "NUMERIC" flag, it is re‐sampled (using the stored distribution) - to preserve the training proportions. + (if the _cat cell equals the DEFAULT_NUMERIC_MARKER) and which rows should directly + receive the special value. + If the generated _cat column contains no DEFAULT_NUMERIC_MARKER flag, it is re‐sampled + using the stored training distribution. """ if not self._model_trained: raise RuntimeError("Syn_Seq aggregator not yet fitted") @@ -183,9 +188,10 @@ def generate_col(self, count: int) -> pd.DataFrame: # Initialize a DataFrame with NaN values for all columns. gen_df = pd.DataFrame({col: [np.nan] * count for col in self._syn_order}) - # (1) Generate the first column using its stored real distribution. + # (1) Generate the first column using its stored distribution. first_col = self._syn_order[0] - if first_col in self._first_col_distribution and len(self._first_col_distribution[first_col]) > 0: + if (first_col in self._first_col_distribution and + len(self._first_col_distribution[first_col]) > 0): gen_df[first_col] = np.random.choice( self._first_col_distribution[first_col], size=count, replace=True ) @@ -196,46 +202,42 @@ def generate_col(self, count: int) -> pd.DataFrame: # (2) For each subsequent column, generate synthetic values. for col in self._syn_order[1:]: method_name = self._method_map.get(col, "cart") - idx = self._syn_order.index(col) - preds_list = self._varsel.get(col, self._syn_order[:idx]) + preds_list = self._varsel.get(col, self._syn_order[:self._syn_order.index(col)]) - # If the column has special values then it has an associated _cat indicator. + # Check if the column has a corresponding _cat indicator. cat_col = col + "_cat" if col in self.special_values and cat_col in gen_df.columns: - # Check if the generated _cat column has any "NUMERIC" flag. - numeric_count = (gen_df[cat_col] == "NUMERIC").sum() + # Check if the generated _cat column has any numeric marker. + numeric_count = (gen_df[cat_col] == DEFAULT_NUMERIC_MARKER).sum() if numeric_count == 0: warnings.warn( - f"Degenerate _cat column for {col}: no rows marked as 'NUMERIC'. " - "Re-sampling _cat column to preserve training distribution." + f"Degenerate _cat column for {col}: no rows marked as numeric. Re-sampling _cat column using training distribution." ) - # Use the stored distribution from training. - cat_dist = self.cat_distributions.get(col, {"NUMERIC": 1.0}) + cat_dist = self.cat_distributions.get(col, {DEFAULT_NUMERIC_MARKER: 1.0}) total = sum(cat_dist.values()) - # Normalize probabilities cat_probs = {k: v / total for k, v in cat_dist.items()} - # Re-sample _cat column: new_cat = np.where( - np.random.rand(count) < cat_probs.get("NUMERIC", 1.0), - "NUMERIC", + np.random.rand(count) < cat_probs.get(DEFAULT_NUMERIC_MARKER, 1.0), + float(DEFAULT_NUMERIC_MARKER), np.random.choice( - [k for k in cat_probs.keys() if k != "NUMERIC"], + [k for k in cat_probs if k != DEFAULT_NUMERIC_MARKER], size=count, - p=[cat_probs[k] for k in cat_probs if k != "NUMERIC"] + p=[cat_probs[k] for k in cat_probs if k != DEFAULT_NUMERIC_MARKER] ) ) gen_df[cat_col] = new_cat + gen_df[cat_col] = gen_df[cat_col].astype(float) - # For rows with _cat equal to "NUMERIC", generate synthetic values. - is_numeric = gen_df[cat_col] == "NUMERIC" + # For rows with _cat equal to the numeric marker, generate synthetic values. + is_numeric = gen_df[cat_col] == DEFAULT_NUMERIC_MARKER if is_numeric.sum() > 0: Xsyn_numeric = gen_df.loc[is_numeric, preds_list].values ysyn_numeric = self._generate_single_col(method_name, Xsyn_numeric, col) gen_df.loc[is_numeric, col] = ysyn_numeric - # For rows where _cat indicates a special value, assign that special value directly. + # For rows where _cat equals a special value, assign that special value directly. for special in self.special_values[col]: - is_special = gen_df[cat_col] == str(special) - gen_df.loc[is_special, col] = float(special) + is_special = gen_df[cat_col] == special + gen_df.loc[is_special, col] = special else: # Otherwise, generate the column normally. Xsyn = gen_df[preds_list].values @@ -247,15 +249,11 @@ def generate_col(self, count: int) -> pd.DataFrame: def _generate_single_col(self, method_name: str, Xsyn: np.ndarray, col: str) -> np.ndarray: """ Generate synthetic values for a single column using the fitted model. + If no model is available for the column, a RuntimeError is raised. """ if col not in self._col_models or self._col_models[col] is None: - if col in self._first_col_distribution and len(self._first_col_distribution[col]) > 0: - return np.random.choice( - self._first_col_distribution[col], size=len(Xsyn), replace=True - ) - else: - return np.zeros(len(Xsyn)) - + raise RuntimeError(f"No model available for column {col}.") fit_info = self._col_models[col] _, generate_func = METHOD_MAP[fit_info["name"]] return generate_func(fit_info["fitted_model"], Xsyn) + diff --git a/src/synthcity/plugins/core/models/syn_seq/syn_seq_preprocess.py b/src/synthcity/plugins/core/models/syn_seq/syn_seq_preprocess.py index d7b6cd1e..0ac867b7 100644 --- a/src/synthcity/plugins/core/models/syn_seq/syn_seq_preprocess.py +++ b/src/synthcity/plugins/core/models/syn_seq/syn_seq_preprocess.py @@ -11,21 +11,28 @@ class SynSeqPreprocessor: - Records the original dtypes. - Automatically assigns dtypes (date/category/numeric) when not provided. - Converts date columns to datetime and category columns to 'category' dtype. - - For numeric columns, if one value accounts for ≥90% of non-null rows, that value is + - For numeric columns, if one value accounts for ≥90% of non‐null rows, that value is automatically marked as a special value. For each such column, a new categorical column (named base_col_cat) is created: - * If the cell value is missing, it is mapped to "NAN". - * If the cell value equals a detected (or user-specified) special value, it is mapped to that special value (as string). - * Otherwise, it is marked as "NUMERIC". + * If the cell value is missing, it is mapped to the missing marker (here, -99999999). + * If the cell value equals a detected (or user‐specified) special value, it is left as its + original numeric value. + * Otherwise, it is marked with the “numeric” marker (here, -999999999) indicating that the + value is not special. Postprocessing: - Merges back the split (base_col, base_col_cat) columns: For rows where the base column is NaN and the corresponding _cat column - indicates a special value (i.e. not "NUMERIC"), the base column is replaced - with that special value. In particular, if _cat equals "NAN", the base column is set to np.nan. - - Optionally applies user-provided rules sequentially to filter rows. + is not equal to the numeric marker, the base column is replaced + with that special value. In particular, if _cat equals the missing marker, + the base column is set to np.nan. + - Optionally applies user‐provided rules sequentially to filter rows. """ + # Define marker constants (choose values unlikely to appear in your data) + NUMERIC_MARKER = -777777777.0 # Indicates a normal (non‐special) value + MISSING_MARKER = -999999999.0 # Indicates a missing value + def __init__( self, user_dtypes: Optional[Dict[str, str]] = None, @@ -35,7 +42,7 @@ def __init__( """ Args: user_dtypes: {col: "date"/"category"/"numeric"}, if not provided, auto-detected. - user_special_values: {col: [special_value1, special_value2, ...]}. + user_special_values: {col: [special_value1, special_value2, ...]}. Even if not provided, special values are detected automatically for imbalanced numeric columns. max_categories: When auto-detecting dtypes, if nunique <= max_categories, assign 'category', else 'numeric'. """ @@ -44,8 +51,8 @@ def __init__( self.max_categories = max_categories # Internal storage - self.original_dtypes: Dict[str, str] = {} # {col: original_dtype} - self.split_map: Dict[str, str] = {} # {base_col -> cat_col} + self.original_dtypes: Dict[str, str] = {} # {col: original_dtype} + self.split_map: Dict[str, str] = {} # {base_col -> cat_col} self.detected_specials: Dict[str, List[Any]] = {} # stores the special values (detected or user-provided) # ========================================================================= @@ -153,11 +160,11 @@ def _detect_special_values(self, df: pd.DataFrame): def _split_numeric_columns(self, df: pd.DataFrame): """ For each numeric column in user_special_values: - - Create a new categorical column (named base_col_cat) that marks special values. + - Create a new numeric column (named base_col_cat) that marks special values using integer markers. - For each cell in the base column: - If NaN -> returns "NAN". - If the value is in the list of special values -> returns that special value (as string). - Otherwise -> returns "NUMERIC". + If NaN -> returns the missing marker (-99999999). + If the value is in the list of special values -> returns that special value. + Otherwise -> returns the numeric marker (-999999999). """ for col, specials in self.user_special_values.items(): if col not in df.columns: @@ -165,24 +172,22 @@ def _split_numeric_columns(self, df: pd.DataFrame): cat_col = col + "_cat" self.split_map[col] = cat_col - # Store the complete list of special values (detected or user provided) self.detected_specials[col] = specials - # Remove any existing cat_col. if cat_col in df.columns: df.drop(columns=[cat_col], inplace=True) base_idx = df.columns.get_loc(col) df.insert(base_idx, cat_col, None) - def cat_mapper(x, specials, normal_marker="NUMERIC", missing_marker="NAN"): + def cat_mapper(x, specials, normal_marker=self.NUMERIC_MARKER, missing_marker=self.MISSING_MARKER): if pd.isna(x): - return missing_marker + return float(missing_marker) elif x in specials: - return str(x) + return float(x) # retain the special value as-is else: - return normal_marker - df[cat_col] = df[col].apply(lambda x: cat_mapper(x, specials)).astype(str) - df[cat_col] = df[cat_col].astype("category") + return float(normal_marker) + + df[cat_col] = df[col].apply(lambda x: cat_mapper(x, specials)) # ========================================================================= # POSTPROCESSING @@ -192,7 +197,7 @@ def postprocess(self, df: pd.DataFrame, rules: Optional[Dict[str, List[Tuple[str Postprocesses the synthetic DataFrame: 1) Merges back split columns (base_col, base_col_cat) by replacing NaNs in the base column with the corresponding special value (if _cat indicates a special value). - In particular, if _cat equals "NAN", the base column is set to np.nan. + In particular, if _cat equals the missing marker, the base column is set to np.nan. 2) Optionally applies user-provided rules sequentially to filter rows. (Note: Date offset restoration is not performed.) """ @@ -207,17 +212,16 @@ def postprocess(self, df: pd.DataFrame, rules: Optional[Dict[str, List[Tuple[str def _merge_splitted_cols(self, df: pd.DataFrame) -> pd.DataFrame: """ For each (base_col, cat_col) pair in split_map: - - If a base column cell is NaN and the corresponding _cat cell is not "NUMERIC", + - If a base column cell is NaN and the corresponding _cat cell is not equal to the numeric marker, then replace the base column cell. - * If the _cat cell is "NAN", set the base column cell to np.nan. - * Otherwise, convert the _cat cell back to its original special value. + * If the _cat cell equals the missing marker, set the base column cell to np.nan. + * Otherwise, set the base column cell to the value in the _cat cell. - Finally, drop the auxiliary _cat column. """ for base_col, cat_col in self.split_map.items(): if base_col in df.columns and cat_col in df.columns: specials = self.detected_specials.get(base_col, []) - # Condition: base column is NaN and _cat is not "NUMERIC" - condition = df[base_col].isna() & (~df[cat_col].isin(["NUMERIC"])) + condition = df[base_col].isna() & (df[cat_col] != self.NUMERIC_MARKER) if condition.any(): df.loc[condition, base_col] = df.loc[condition, cat_col].apply( lambda v: self._convert_special_value(v, specials) @@ -225,16 +229,15 @@ def _merge_splitted_cols(self, df: pd.DataFrame) -> pd.DataFrame: df.drop(columns=[cat_col], inplace=True) return df - def _convert_special_value(self, val: str, specials: List[Any]) -> Any: + def _convert_special_value(self, val: Any, specials: List[Any]) -> Any: """ - Given the string representation of a special value and the list of original special values, - returns the original special value. In particular, if val equals "NAN", return np.nan. + Given the numeric marker from a _cat column and the list of original special values, + returns the original special value. In particular, if val equals the missing marker, + returns np.nan. """ - if val == "NAN": + if val == self.MISSING_MARKER: return np.nan - for special in specials: - if str(special) == val: - return special + # If the value is one of the special values, return it; otherwise, return the value as-is. return val def apply_rules(self, df: pd.DataFrame, rules: Dict[str, List[Tuple[str, str, Any]]]) -> pd.DataFrame: diff --git a/tutorials/tutorial10_sequential_synthesis.ipynb b/tutorials/tutorial10_sequential_synthesis.ipynb index 91a58c3e..8d62e259 100644 --- a/tutorials/tutorial10_sequential_synthesis.ipynb +++ b/tutorials/tutorial10_sequential_synthesis.ipynb @@ -24,9 +24,188 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: synthcity in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (0.2.11)\n", + "Requirement already satisfied: importlib-metadata in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (8.5.0)\n", + "Requirement already satisfied: pandas>=2.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.2.3)\n", + "Requirement already satisfied: torch<2.3,>=2.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.2.2)\n", + "Requirement already satisfied: scikit-learn>=1.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (1.6.0)\n", + "Requirement already satisfied: nflows>=0.14 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.14)\n", + "Requirement already satisfied: numpy<2.0,>=1.20 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (1.26.4)\n", + "Requirement already satisfied: lifelines<0.30.0,>=0.29.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.29.0)\n", + "Requirement already satisfied: opacus>=1.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (1.5.2)\n", + "Requirement already satisfied: networkx<3.0,>2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.8.8)\n", + "Requirement already satisfied: decaf-synthetic-data>=0.1.6 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.1.6)\n", + "Requirement already satisfied: optuna>=3.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (4.1.0)\n", + "Requirement already satisfied: shap in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.46.0)\n", + "Requirement already satisfied: tenacity in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (9.0.0)\n", + "Requirement already satisfied: tqdm in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (4.67.1)\n", + "Requirement already satisfied: loguru in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.7.3)\n", + "Requirement already satisfied: pydantic>=2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.10.4)\n", + "Requirement already satisfied: cloudpickle in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (3.1.0)\n", + "Requirement already satisfied: scipy in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (1.13.1)\n", + "Requirement already satisfied: xgboost<3.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.1.3)\n", + "Requirement already satisfied: geomloss in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.2.6)\n", + "Requirement already satisfied: pgmpy in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.1.26)\n", + "Requirement already satisfied: redis in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (5.2.1)\n", + "Requirement already satisfied: pycox in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.3.0)\n", + "Requirement already satisfied: xgbse>=0.3.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.3.3)\n", + "Requirement already satisfied: pykeops in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (2.2.3)\n", + "Requirement already satisfied: fflows in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.0.3)\n", + "Requirement already satisfied: monai in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (1.4.0)\n", + "Requirement already satisfied: tsai in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.3.9)\n", + "Requirement already satisfied: be-great>=0.0.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.0.8)\n", + "Requirement already satisfied: arfpy in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from synthcity) (0.1.1)\n", + "Requirement already satisfied: datasets>=2.5.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from be-great>=0.0.5->synthcity) (3.2.0)\n", + "Requirement already satisfied: transformers>=4.22.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from be-great>=0.0.5->synthcity) (4.47.1)\n", + "Requirement already satisfied: accelerate>=0.20.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from be-great>=0.0.5->synthcity) (1.2.1)\n", + "Requirement already satisfied: pytorch-lightning<2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from decaf-synthetic-data>=0.1.6->synthcity) (1.9.5)\n", + "Requirement already satisfied: torchtext>=0.10 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from decaf-synthetic-data>=0.1.6->synthcity) (0.17.2)\n", + "Requirement already satisfied: matplotlib>=3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from lifelines<0.30.0,>=0.29.0->synthcity) (3.9.4)\n", + "Requirement already satisfied: autograd>=1.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from lifelines<0.30.0,>=0.29.0->synthcity) (1.7.0)\n", + "Requirement already satisfied: autograd-gamma>=0.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from lifelines<0.30.0,>=0.29.0->synthcity) (0.5.0)\n", + "Requirement already satisfied: formulaic>=0.2.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from lifelines<0.30.0,>=0.29.0->synthcity) (1.1.1)\n", + "Requirement already satisfied: tensorboard in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from nflows>=0.14->synthcity) (2.18.0)\n", + "Requirement already satisfied: opt-einsum>=3.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from opacus>=1.3->synthcity) (3.4.0)\n", + "Requirement already satisfied: alembic>=1.5.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from optuna>=3.1->synthcity) (1.14.0)\n", + "Requirement already satisfied: colorlog in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from optuna>=3.1->synthcity) (6.9.0)\n", + "Requirement already satisfied: packaging>=20.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from optuna>=3.1->synthcity) (24.2)\n", + "Requirement already satisfied: sqlalchemy>=1.4.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from optuna>=3.1->synthcity) (2.0.36)\n", + "Requirement already satisfied: PyYAML in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from optuna>=3.1->synthcity) (6.0.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pandas>=2.1->synthcity) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pandas>=2.1->synthcity) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pandas>=2.1->synthcity) (2024.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pydantic>=2.0->synthcity) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pydantic>=2.0->synthcity) (2.27.2)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pydantic>=2.0->synthcity) (4.12.2)\n", + "Requirement already satisfied: joblib>=1.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from scikit-learn>=1.2->synthcity) (1.4.2)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from scikit-learn>=1.2->synthcity) (3.5.0)\n", + "Requirement already satisfied: filelock in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from torch<2.3,>=2.1->synthcity) (3.16.1)\n", + "Requirement already satisfied: sympy in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from torch<2.3,>=2.1->synthcity) (1.13.3)\n", + "Requirement already satisfied: jinja2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from torch<2.3,>=2.1->synthcity) (3.1.5)\n", + "Requirement already satisfied: fsspec in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from torch<2.3,>=2.1->synthcity) (2024.9.0)\n", + "Requirement already satisfied: zipp>=3.20 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from importlib-metadata->synthcity) (3.21.0)\n", + "Requirement already satisfied: pyparsing in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pgmpy->synthcity) (3.2.1)\n", + "Requirement already satisfied: statsmodels in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pgmpy->synthcity) (0.14.4)\n", + "Requirement already satisfied: google-generativeai in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pgmpy->synthcity) (0.8.3)\n", + "Requirement already satisfied: torchtuples>=0.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (0.2.2)\n", + "Requirement already satisfied: feather-format>=0.4.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (0.4.1)\n", + "Requirement already satisfied: h5py>=2.9.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (3.12.1)\n", + "Requirement already satisfied: numba>=0.44 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (0.60.0)\n", + "Requirement already satisfied: requests>=2.22.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (2.32.3)\n", + "Requirement already satisfied: py7zr>=0.11.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pycox->synthcity) (0.22.0)\n", + "Requirement already satisfied: pybind11 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pykeops->synthcity) (2.13.6)\n", + "Requirement already satisfied: keopscore==2.2.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pykeops->synthcity) (2.2.3)\n", + "Requirement already satisfied: async-timeout>=4.0.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from redis->synthcity) (5.0.1)\n", + "Requirement already satisfied: slicer==0.0.8 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from shap->synthcity) (0.0.8)\n", + "Requirement already satisfied: fastai>=2.7.14 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tsai->synthcity) (2.7.18)\n", + "Requirement already satisfied: pyts>=0.12.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tsai->synthcity) (0.13.0)\n", + "Requirement already satisfied: imbalanced-learn>=0.11.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tsai->synthcity) (0.12.4)\n", + "Requirement already satisfied: psutil>=5.4.8 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tsai->synthcity) (5.8.0)\n", + "Requirement already satisfied: huggingface-hub>=0.21.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from accelerate>=0.20.1->be-great>=0.0.5->synthcity) (0.27.0)\n", + "Requirement already satisfied: safetensors>=0.4.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from accelerate>=0.20.1->be-great>=0.0.5->synthcity) (0.5.0)\n", + "Requirement already satisfied: Mako in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from alembic>=1.5.0->optuna>=3.1->synthcity) (1.3.8)\n", + "Requirement already satisfied: pyarrow>=15.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from datasets>=2.5.2->be-great>=0.0.5->synthcity) (18.1.0)\n", + "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from datasets>=2.5.2->be-great>=0.0.5->synthcity) (0.3.8)\n", + "Requirement already satisfied: xxhash in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from datasets>=2.5.2->be-great>=0.0.5->synthcity) (3.5.0)\n", + "Requirement already satisfied: multiprocess<0.70.17 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from datasets>=2.5.2->be-great>=0.0.5->synthcity) (0.70.16)\n", + "Requirement already satisfied: aiohttp in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from datasets>=2.5.2->be-great>=0.0.5->synthcity) (3.11.11)\n", + "Requirement already satisfied: pip in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (24.2)\n", + "Requirement already satisfied: fastdownload<2,>=0.0.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (0.0.7)\n", + "Requirement already satisfied: fastcore<1.8,>=1.5.29 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (1.7.28)\n", + "Requirement already satisfied: torchvision>=0.11 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (0.17.2)\n", + "Requirement already satisfied: fastprogress>=0.2.4 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (1.0.3)\n", + "Requirement already satisfied: pillow>=9.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (11.1.0)\n", + "Requirement already satisfied: spacy<4 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from fastai>=2.7.14->tsai->synthcity) (3.8.3)\n", + "Requirement already satisfied: interface-meta>=1.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from formulaic>=0.2.2->lifelines<0.30.0,>=0.29.0->synthcity) (1.3.0)\n", + "Requirement already satisfied: wrapt>=1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from formulaic>=0.2.2->lifelines<0.30.0,>=0.29.0->synthcity) (1.17.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from matplotlib>=3.0->lifelines<0.30.0,>=0.29.0->synthcity) (1.3.0)\n", + "Requirement already satisfied: cycler>=0.10 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from matplotlib>=3.0->lifelines<0.30.0,>=0.29.0->synthcity) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from matplotlib>=3.0->lifelines<0.30.0,>=0.29.0->synthcity) (4.55.3)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from matplotlib>=3.0->lifelines<0.30.0,>=0.29.0->synthcity) (1.4.7)\n", + "Requirement already satisfied: importlib-resources>=3.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from matplotlib>=3.0->lifelines<0.30.0,>=0.29.0->synthcity) (6.5.2)\n", + "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from numba>=0.44->pycox->synthcity) (0.43.0)\n", + "Requirement already satisfied: texttable in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (1.7.0)\n", + "Requirement already satisfied: pycryptodomex>=3.16.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (3.21.0)\n", + "Requirement already satisfied: pyzstd>=0.15.9 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (0.16.2)\n", + "Requirement already satisfied: pyppmd<1.2.0,>=1.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (1.1.1)\n", + "Requirement already satisfied: pybcj<1.1.0,>=1.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (1.0.3)\n", + "Requirement already satisfied: multivolumefile>=0.2.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (0.2.3)\n", + "Requirement already satisfied: inflate64<1.1.0,>=1.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (1.0.1)\n", + "Requirement already satisfied: brotli>=1.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from py7zr>=0.11.3->pycox->synthcity) (1.1.0)\n", + "Requirement already satisfied: six>=1.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas>=2.1->synthcity) (1.17.0)\n", + "Requirement already satisfied: torchmetrics>=0.7.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pytorch-lightning<2.0->decaf-synthetic-data>=0.1.6->synthcity) (1.6.1)\n", + "Requirement already satisfied: lightning-utilities>=0.6.0.post0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pytorch-lightning<2.0->decaf-synthetic-data>=0.1.6->synthcity) (0.11.9)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from requests>=2.22.0->pycox->synthcity) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from requests>=2.22.0->pycox->synthcity) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from requests>=2.22.0->pycox->synthcity) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from requests>=2.22.0->pycox->synthcity) (2024.12.14)\n", + "Requirement already satisfied: regex!=2019.12.17 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from transformers>=4.22.1->be-great>=0.0.5->synthcity) (2024.11.6)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from transformers>=4.22.1->be-great>=0.0.5->synthcity) (0.21.0)\n", + "Requirement already satisfied: google-ai-generativelanguage==0.6.10 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-generativeai->pgmpy->synthcity) (0.6.10)\n", + "Requirement already satisfied: google-api-core in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-generativeai->pgmpy->synthcity) (2.24.0)\n", + "Requirement already satisfied: google-api-python-client in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-generativeai->pgmpy->synthcity) (2.157.0)\n", + "Requirement already satisfied: google-auth>=2.15.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-generativeai->pgmpy->synthcity) (2.37.0)\n", + "Requirement already satisfied: protobuf in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-generativeai->pgmpy->synthcity) (5.29.2)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-ai-generativelanguage==0.6.10->google-generativeai->pgmpy->synthcity) (1.25.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from jinja2->torch<2.3,>=2.1->synthcity) (3.0.2)\n", + "Requirement already satisfied: patsy>=0.5.6 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from statsmodels->pgmpy->synthcity) (1.0.1)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from sympy->torch<2.3,>=2.1->synthcity) (1.3.0)\n", + "Requirement already satisfied: absl-py>=0.4 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (2.1.0)\n", + "Requirement already satisfied: grpcio>=1.48.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (1.69.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (3.7)\n", + "Requirement already satisfied: setuptools>=41.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (75.1.0)\n", + "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (0.7.2)\n", + "Requirement already satisfied: werkzeug>=1.0.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from tensorboard->nflows>=0.14->synthcity) (3.1.3)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (2.4.4)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (1.3.2)\n", + "Requirement already satisfied: attrs>=17.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (24.3.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (1.5.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (0.2.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from aiohttp->datasets>=2.5.2->be-great>=0.0.5->synthcity) (1.18.3)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-api-core->google-generativeai->pgmpy->synthcity) (1.66.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-auth>=2.15.0->google-generativeai->pgmpy->synthcity) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-auth>=2.15.0->google-generativeai->pgmpy->synthcity) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-auth>=2.15.0->google-generativeai->pgmpy->synthcity) (4.9)\n", + "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (3.0.12)\n", + "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (1.0.5)\n", + "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (1.0.11)\n", + "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (2.0.10)\n", + "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (3.0.9)\n", + "Requirement already satisfied: thinc<8.4.0,>=8.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (8.3.3)\n", + "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (1.1.3)\n", + "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (2.5.0)\n", + "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (2.0.10)\n", + "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (0.4.1)\n", + "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (0.15.1)\n", + "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from spacy<4->fastai>=2.7.14->tsai->synthcity) (3.5.0)\n", + "Requirement already satisfied: httplib2<1.dev0,>=0.19.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-api-python-client->google-generativeai->pgmpy->synthcity) (0.22.0)\n", + "Requirement already satisfied: google-auth-httplib2<1.0.0,>=0.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-api-python-client->google-generativeai->pgmpy->synthcity) (0.2.0)\n", + "Requirement already satisfied: uritemplate<5,>=3.0.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-api-python-client->google-generativeai->pgmpy->synthcity) (4.1.1)\n", + "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-ai-generativelanguage==0.6.10->google-generativeai->pgmpy->synthcity) (1.69.0)\n", + "Requirement already satisfied: language-data>=1.2 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from langcodes<4.0.0,>=3.2.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (1.3.0)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth>=2.15.0->google-generativeai->pgmpy->synthcity) (0.6.1)\n", + "Requirement already satisfied: blis<1.2.0,>=1.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from thinc<8.4.0,>=8.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (1.1.0)\n", + "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from thinc<8.4.0,>=8.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (0.1.5)\n", + "Requirement already satisfied: click>=8.0.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (8.1.8)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (1.5.4)\n", + "Requirement already satisfied: rich>=10.11.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (13.9.4)\n", + "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from weasel<0.5.0,>=0.1.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (0.20.0)\n", + "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from weasel<0.5.0,>=0.1.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (7.1.0)\n", + "Requirement already satisfied: marisa-trie>=1.1.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (1.2.1)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (2.18.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /Users/minkeychang/anaconda3/envs/syn_seq/lib/python3.9/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<4->fastai>=2.7.14->tsai->synthcity) (0.1.2)\n" + ] + } + ], "source": [ "!pip install synthcity" ] @@ -519,8 +698,8 @@ "data": { "text/plain": [ "capital-gain_cat\n", - "0 44888\n", - "NUMERIC 3954\n", + " 0.0 44888\n", + "-777777777.0 3954\n", "Name: count, dtype: int64" ] }, @@ -542,8 +721,8 @@ "data": { "text/plain": [ "capital-loss_cat\n", - "0 46560\n", - "NUMERIC 2282\n", + " 0.0 46560\n", + "-777777777.0 2282\n", "Name: count, dtype: int64" ] }, @@ -806,9 +985,9 @@ " 3\n", " 0\n", " 1\n", - " NUMERIC\n", + " -777777777.0\n", " 2\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 0\n", @@ -825,9 +1004,9 @@ " 2\n", " 0\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 12\n", " 0\n", @@ -844,9 +1023,9 @@ " 3\n", " 0\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 0\n", @@ -863,9 +1042,9 @@ " 2\n", " 4\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 0\n", @@ -882,9 +1061,9 @@ " 0\n", " 4\n", " 0\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 12\n", @@ -920,9 +1099,9 @@ " 3\n", " 0\n", " 0\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 35\n", " 0\n", @@ -939,9 +1118,9 @@ " 4\n", " 4\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 0\n", @@ -958,9 +1137,9 @@ " 2\n", " 0\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 49\n", " 0\n", @@ -977,9 +1156,9 @@ " 1\n", " 1\n", " 1\n", - " NUMERIC\n", + " -777777777.0\n", " 5\n", - " 0\n", + " 0.0\n", " 0\n", " 39\n", " 0\n", @@ -996,9 +1175,9 @@ " 2\n", " 0\n", " 1\n", + " 0.0\n", " 0\n", - " 0\n", - " 0\n", + " 0.0\n", " 0\n", " 59\n", " 0\n", @@ -1023,18 +1202,18 @@ "48840 28 0 4 12 1 8 \n", "48841 19 2 11 12 0 4 \n", "\n", - " relationship race sex capital-gain_cat capital-gain capital-loss_cat \\\n", - "0 3 0 1 NUMERIC 2 0 \n", - "1 2 0 1 0 0 0 \n", - "2 3 0 1 0 0 0 \n", - "3 2 4 1 0 0 0 \n", - "4 0 4 0 0 0 0 \n", - "... ... ... .. ... ... ... \n", - "48837 3 0 0 0 0 0 \n", - "48838 4 4 1 0 0 0 \n", - "48839 2 0 1 0 0 0 \n", - "48840 1 1 1 NUMERIC 5 0 \n", - "48841 2 0 1 0 0 0 \n", + " relationship race sex capital-gain_cat capital-gain capital-loss_cat \\\n", + "0 3 0 1 -777777777.0 2 0.0 \n", + "1 2 0 1 0.0 0 0.0 \n", + "2 3 0 1 0.0 0 0.0 \n", + "3 2 4 1 0.0 0 0.0 \n", + "4 0 4 0 0.0 0 0.0 \n", + "... ... ... .. ... ... ... \n", + "48837 3 0 0 0.0 0 0.0 \n", + "48838 4 4 1 0.0 0 0.0 \n", + "48839 2 0 1 0.0 0 0.0 \n", + "48840 1 1 1 -777777777.0 5 0.0 \n", + "48841 2 0 1 0.0 0 0.0 \n", "\n", " capital-loss hours-per-week native-country income>50K \n", "0 0 39 0 0 \n", @@ -1081,41 +1260,41 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-02-19T21:49:08.588848+0900][1593][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n", - "[2025-02-19T21:49:08.588848+0900][1593][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n" + "[2025-02-20T02:06:22.336913+0900][4686][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n", + "[2025-02-20T02:06:22.336913+0900][4686][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n" ] }, { "data": { "text/plain": [ "['uniform_sampler',\n", - " 'survival_nflow',\n", - " 'image_cgan',\n", " 'ctgan',\n", - " 'bayesian_network',\n", - " 'survival_ctgan',\n", - " 'timegan',\n", - " 'decaf',\n", - " 'aim',\n", - " 'privbayes',\n", " 'arf',\n", + " 'marginal_distributions',\n", + " 'privbayes',\n", + " 'tvae',\n", + " 'image_cgan',\n", + " 'aim',\n", " 'radialgan',\n", - " 'timevae',\n", - " 'survival_gan',\n", - " 'pategan',\n", - " 'dummy_sampler',\n", " 'survae',\n", - " 'marginal_distributions',\n", " 'rtvae',\n", + " 'decaf',\n", + " 'pategan',\n", + " 'ddpm',\n", + " 'survival_nflow',\n", + " 'survival_ctgan',\n", + " 'timevae',\n", + " 'dummy_sampler',\n", " 'great',\n", - " 'dpgan',\n", + " 'survival_gan',\n", " 'nflow',\n", - " 'ddpm',\n", - " 'image_adsgan',\n", - " 'syn_seq',\n", " 'adsgan',\n", - " 'tvae',\n", - " 'fflows']" + " 'dpgan',\n", + " 'timegan',\n", + " 'fflows',\n", + " 'bayesian_network',\n", + " 'syn_seq',\n", + " 'image_adsgan']" ] }, "execution_count": 12, @@ -1141,7 +1320,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2025-02-19T21:49:10.199391+0900][1593][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n" + "[2025-02-20T02:06:23.344137+0900][4686][CRITICAL] module disabled: /Users/minkeychang/synthcity/src/synthcity/plugins/generic/plugin_goggle.py\n" ] } ], @@ -1180,7 +1359,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -1194,7 +1373,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -1218,6 +1397,23 @@ "Generating 'capital-gain_cat' => done.\n", "Generating 'capital-gain' => done.\n" ] + }, + { + "ename": "ValueError", + "evalue": "y contains previously unseen labels: [-777777777]", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m synthetic_loader \u001b[38;5;241m=\u001b[39m \u001b[43msyn_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/syn_seq/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:55\u001b[0m, in \u001b[0;36mvalidate_arguments..validate..wrapper_function\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(_func)\n\u001b[1;32m 54\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper_function\u001b[39m(\u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m---> 55\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mvd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/syn_seq/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:150\u001b[0m, in \u001b[0;36mValidatedFunction.call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[1;32m 149\u001b[0m m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minit_model_instance(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m--> 150\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mm\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/syn_seq/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:226\u001b[0m, in \u001b[0;36mValidatedFunction.execute\u001b[0;34m(self, m)\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw_function(\u001b[38;5;241m*\u001b[39margs_, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mvar_kwargs)\n\u001b[1;32m 225\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43md\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mvar_kwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/synthcity/src/synthcity/plugins/core/plugin.py:355\u001b[0m, in \u001b[0;36mPlugin.generate\u001b[0;34m(self, count, constraints, random_state, **kwargs)\u001b[0m\n\u001b[1;32m 353\u001b[0m X_syn \u001b[38;5;241m=\u001b[39m X_syn\u001b[38;5;241m.\u001b[39mdecompress(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompress_context)\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data_encoders \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 355\u001b[0m X_syn \u001b[38;5;241m=\u001b[39m \u001b[43mX_syn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_data_encoders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 357\u001b[0m \u001b[38;5;66;03m# The dataset is decompressed here, we can use the public schema\u001b[39;00m\n\u001b[1;32m 358\u001b[0m gen_constraints \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mschema()\u001b[38;5;241m.\u001b[39mas_constraints()\n", + "File \u001b[0;32m~/synthcity/src/synthcity/plugins/core/dataloader.py:261\u001b[0m, in \u001b[0;36mDataLoader.decode\u001b[0;34m(self, encoders)\u001b[0m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 259\u001b[0m decoded[col] \u001b[38;5;241m=\u001b[39m decoded[col]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mfloat\u001b[39m)\n\u001b[0;32m--> 261\u001b[0m decoded[col] \u001b[38;5;241m=\u001b[39m \u001b[43mencoders\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcol\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minverse_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecoded\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcol\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfrom_info(decoded, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minfo())\n", + "File \u001b[0;32m~/anaconda3/envs/syn_seq/lib/python3.9/site-packages/sklearn/preprocessing/_label.py:162\u001b[0m, in \u001b[0;36mLabelEncoder.inverse_transform\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m 156\u001b[0m diff \u001b[38;5;241m=\u001b[39m _setdiff1d(\n\u001b[1;32m 157\u001b[0m ar1\u001b[38;5;241m=\u001b[39my,\n\u001b[1;32m 158\u001b[0m ar2\u001b[38;5;241m=\u001b[39mxp\u001b[38;5;241m.\u001b[39marange(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclasses_\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m], device\u001b[38;5;241m=\u001b[39mdevice(y)),\n\u001b[1;32m 159\u001b[0m xp\u001b[38;5;241m=\u001b[39mxp,\n\u001b[1;32m 160\u001b[0m )\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m diff\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]:\n\u001b[0;32m--> 162\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124my contains previously unseen labels: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m \u001b[38;5;28mstr\u001b[39m(diff))\n\u001b[1;32m 163\u001b[0m y \u001b[38;5;241m=\u001b[39m xp\u001b[38;5;241m.\u001b[39masarray(y)\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m xp\u001b[38;5;241m.\u001b[39mtake(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclasses_, y, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n", + "\u001b[0;31mValueError\u001b[0m: y contains previously unseen labels: [-777777777]" + ] } ], "source": [ @@ -1228,504 +1424,19 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
agesexworkclasseducation-nummarital-statusoccupationrelationshipfnlwgtracecapital-loss_catcapital-losshours-per-weeknative-countryincome>50Kcapital-gain_catcapital-gain
09104062220050390007
131101203270048390102
225008080120039590004
3451880142110037390004
449088414384034390002
5401212032130038490107
618101205260046490104
722008040110038540003
81005231800481400015
92710901220032490008
107109261120038190002
118051325390033190007
1211082242000321900010
1360011283120033140007
1421051315570045390007
1550080702240313600015
1631108012400343900015
178108072100021410003
1822106173500393900015
195059281190032140004
\n", - "
" - ], - "text/plain": [ - " age sex workclass education-num marital-status occupation \\\n", - "0 9 1 0 4 0 6 \n", - "1 31 1 0 12 0 3 \n", - "2 25 0 0 8 0 8 \n", - "3 45 1 8 8 0 14 \n", - "4 49 0 8 8 4 14 \n", - "5 40 1 2 12 0 3 \n", - "6 18 1 0 12 0 5 \n", - "7 22 0 0 8 0 4 \n", - "8 1 0 0 5 2 3 \n", - "9 27 1 0 9 0 1 \n", - "10 7 1 0 9 2 6 \n", - "11 8 0 5 13 2 5 \n", - "12 1 1 0 8 2 2 \n", - "13 6 0 0 11 2 8 \n", - "14 21 0 5 13 1 5 \n", - "15 5 0 0 8 0 7 \n", - "16 31 1 0 8 0 1 \n", - "17 8 1 0 8 0 7 \n", - "18 22 1 0 6 1 7 \n", - "19 5 0 5 9 2 8 \n", - "\n", - " relationship fnlwgt race capital-loss_cat capital-loss hours-per-week \\\n", - "0 2 22 0 0 50 39 \n", - "1 2 7 0 0 48 39 \n", - "2 0 12 0 0 39 59 \n", - "3 2 11 0 0 37 39 \n", - "4 3 8 4 0 34 39 \n", - "5 2 13 0 0 38 49 \n", - "6 2 6 0 0 46 49 \n", - "7 0 11 0 0 38 54 \n", - "8 1 8 0 0 48 14 \n", - "9 2 2 0 0 32 49 \n", - "10 1 12 0 0 38 19 \n", - "11 3 9 0 0 33 19 \n", - "12 4 20 0 0 32 19 \n", - "13 3 12 0 0 33 14 \n", - "14 5 7 0 0 45 39 \n", - "15 0 22 4 0 31 36 \n", - "16 2 4 0 0 34 39 \n", - "17 2 10 0 0 21 41 \n", - "18 3 5 0 0 39 39 \n", - "19 1 19 0 0 32 14 \n", - "\n", - " native-country income>50K capital-gain_cat capital-gain \n", - "0 0 0 0 7 \n", - "1 0 1 0 2 \n", - "2 0 0 0 4 \n", - "3 0 0 0 4 \n", - "4 0 0 0 2 \n", - "5 0 1 0 7 \n", - "6 0 1 0 4 \n", - "7 0 0 0 3 \n", - "8 0 0 0 15 \n", - "9 0 0 0 8 \n", - "10 0 0 0 2 \n", - "11 0 0 0 7 \n", - "12 0 0 0 10 \n", - "13 0 0 0 7 \n", - "14 0 0 0 7 \n", - "15 0 0 0 15 \n", - "16 0 0 0 15 \n", - "17 0 0 0 3 \n", - "18 0 0 0 15 \n", - "19 0 0 0 4 " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" + "ename": "NameError", + "evalue": "name 'synthetic_loader' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m synthetic_df \u001b[38;5;241m=\u001b[39m \u001b[43msynthetic_loader\u001b[49m\u001b[38;5;241m.\u001b[39mdataframe()\n\u001b[1;32m 2\u001b[0m synthetic_df\u001b[38;5;241m.\u001b[39mhead(\u001b[38;5;241m20\u001b[39m)\n", + "\u001b[0;31mNameError\u001b[0m: name 'synthetic_loader' is not defined" + ] } ], "source": [ @@ -1735,7 +1446,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [ {