sklearn.preprocessing Complete Guide: Data Scaling, Encoding & Transformation in Python

Raw Data → Clean Data → Transformed Data → ML Model

1. Feature Scaling      → Numbers ka range equalize karna
2. Encoding            → Strings ko numbers mein convert karna  
3. Normalization       → Rows ko unit norm pe laana
4. Transformation      → Distribution shape change karna
5. Binarization        → Continuous values ko 0/1 mein convert karna
6. Imputation          → Missing values fill karna (SimpleImputer)
7. Polynomial Features → New features generate karna

# Pura module import karo
from sklearn import preprocessing

# Ya specific classes import karo (recommended)
from sklearn.preprocessing import (
    StandardScaler,
    MinMaxScaler,
    MaxAbsScaler,
    RobustScaler,
    Normalizer,
    Binarizer,
    LabelEncoder,
    OrdinalEncoder,
    OneHotEncoder,
    LabelBinarizer,
    MultiLabelBinarizer,
    PolynomialFeatures,
    FunctionTransformer,
    PowerTransformer,
    QuantileTransformer,
    KBinsDiscretizer,
)

# Step 1: Object banao
scaler = StandardScaler()

# Step 2: Training data pe fit karo (statistics seekhta hai)
scaler.fit(X_train)

# Step 3: Transform karo
X_train_scaled = scaler.transform(X_train)
X_test_scaled  = scaler.transform(X_test)  # Same scaler use karo!

# Ya fit + transform ek saath (sirf training data pe)
X_train_scaled = scaler.fit_transform(X_train)

from sklearn.preprocessing import StandardScaler
import numpy as np

data = np.array([[1000, 2], [2000, 4], [3000, 6], [4000, 8]])

scaler = StandardScaler()
scaled = scaler.fit_transform(data)

print("Original:\n", data)
print("\nScaled:\n", scaled)
print("\nMean:", scaler.mean_)
print("Std:", scaler.scale_)

Original:
 [[1000    2]
  [2000    4]
  [3000    6]
  [4000    8]]

Scaled:
 [[-1.34164079 -1.34164079]
  [-0.4472136  -0.4472136 ]
  [ 0.4472136   0.4472136 ]
  [ 1.34164079  1.34164079]]

Mean: [2500.    5.]
Std:  [1118.03398875    2.23606798]

from sklearn.preprocessing import MinMaxScaler

data = np.array([[100], [200], [300], [400], [500]])

scaler = MinMaxScaler(feature_range=(0, 1))  # Default
scaled = scaler.fit_transform(data)
print(scaled)
# [[0.  ], [0.25], [0.5 ], [0.75], [1.  ]]

# Custom range: -1 to 1
scaler2 = MinMaxScaler(feature_range=(-1, 1))
scaled2 = scaler2.fit_transform(data)
print(scaled2)
# [[-1. ], [-0.5], [0.  ], [0.5 ], [1.  ]]

from sklearn.preprocessing import MaxAbsScaler

data = np.array([[-10], [-5], [0], [5], [10]])

scaler = MaxAbsScaler()
scaled = scaler.fit_transform(data)
print(scaled)
# [[-1. ], [-0.5], [0. ], [0.5], [1. ]]

from sklearn.preprocessing import RobustScaler

# Outliers ke saath data
data = np.array([[1], [2], [3], [4], [100]])  # 100 outlier hai

scaler = RobustScaler()
scaled = scaler.fit_transform(data)
print(scaled)
# [[-1.  ], [-0.5 ], [0.   ], [0.5  ], [48.5 ]]

print("Median:", scaler.center_)   # [3.]
print("IQR:", scaler.scale_)       # [2.]

from sklearn.preprocessing import Normalizer

data = np.array([[3, 4], [1, 2], [6, 8]])

# L2 norm (default): sqrt(sum of squares) = 1
norm_l2 = Normalizer(norm='l2')
print(norm_l2.fit_transform(data))
# [[0.6, 0.8], [0.447, 0.894], [0.6, 0.8]]

# L1 norm: sum of absolute values = 1
norm_l1 = Normalizer(norm='l1')
print(norm_l1.fit_transform(data))
# [[0.429, 0.571], [0.333, 0.667], [0.429, 0.571]]

# Max norm: max value = 1
norm_max = Normalizer(norm='max')
print(norm_max.fit_transform(data))
# [[0.75, 1.0], [0.5, 1.0], [0.75, 1.0]]

from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()

# String labels
cities = ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai', 'Delhi']
encoded = le.fit_transform(cities)
print(encoded)       # [2 1 0 2 1]
print(le.classes_)   # ['Bangalore' 'Delhi' 'Mumbai']

# Inverse transform
print(le.inverse_transform([2, 1, 0]))  # ['Mumbai' 'Delhi' 'Bangalore']

from sklearn.preprocessing import OrdinalEncoder

data = np.array([
    ['Low', 'Small'],
    ['Medium', 'Medium'],
    ['High', 'Large'],
    ['Low', 'Large']
])

encoder = OrdinalEncoder(
    categories=[
        ['Low', 'Medium', 'High'],    # Column 1 order
        ['Small', 'Medium', 'Large']   # Column 2 order
    ]
)

encoded = encoder.fit_transform(data)
print(encoded)
# [[0. 0.]
#  [1. 1.]
#  [2. 2.]
#  [0. 2.]]

from sklearn.preprocessing import OneHotEncoder
import pandas as pd

data = np.array([['Mumbai'], ['Delhi'], ['Bangalore'], ['Mumbai']])

ohe = OneHotEncoder(
    sparse_output=False,    # numpy array return karo
    drop='first',           # Dummy variable trap avoid karo
    handle_unknown='ignore' # New categories ko ignore karo
)

encoded = ohe.fit_transform(data)
print(encoded)
# [[0. 1.]    <- Mumbai
#  [1. 0.]    <- Delhi  
#  [0. 0.]    <- Bangalore (dropped - first)
#  [0. 1.]]   <- Mumbai

print(ohe.categories_)
print(ohe.get_feature_names_out(['city']))
# ['city_Delhi' 'city_Mumbai']

from sklearn.preprocessing import LabelBinarizer

lb = LabelBinarizer()
cities = ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai']

encoded = lb.fit_transform(cities)
print(encoded)
# [[0 0 1]   <- Mumbai
#  [0 1 0]   <- Delhi
#  [1 0 0]   <- Bangalore
#  [0 0 1]]  <- Mumbai

print(lb.classes_)  # ['Bangalore' 'Delhi' 'Mumbai']

from sklearn.preprocessing import MultiLabelBinarizer

mlb = MultiLabelBinarizer()

# Ek user multiple genres pasand karta hai
genres = [
    ['Action', 'Drama'],
    ['Comedy'],
    ['Action', 'Comedy', 'Romance'],
    ['Drama', 'Romance']
]

encoded = mlb.fit_transform(genres)
print(mlb.classes_)  # ['Action' 'Comedy' 'Drama' 'Romance']
print(encoded)
# [[1 0 1 0]
#  [0 1 0 0]
#  [1 1 0 1]
#  [0 0 1 1]]

from sklearn.preprocessing import Binarizer

data = np.array([[0.1, 0.5, 0.9], [0.2, 0.6, 0.4]])

# Default threshold = 0.0
binarizer = Binarizer(threshold=0.5)
result = binarizer.fit_transform(data)
print(result)
# [[0 0 1]
#  [0 1 0]]

from sklearn.preprocessing import PowerTransformer

data = np.array([[1], [2], [3], [4], [100], [1000]])  # Skewed data

# Yeo-Johnson: positive aur negative dono values handle karta hai
pt_yj = PowerTransformer(method='yeo-johnson')
transformed = pt_yj.fit_transform(data)
print("Yeo-Johnson:\n", transformed)

# Box-Cox: sirf positive values
pt_bc = PowerTransformer(method='box-cox')
transformed_bc = pt_bc.fit_transform(data)
print("Box-Cox lambda:", pt_bc.lambdas_)

from sklearn.preprocessing import QuantileTransformer

data = np.array([[1], [2], [3], [100], [1000], [50000]])

# Uniform distribution (0 to 1)
qt_uniform = QuantileTransformer(
    n_quantiles=100,
    output_distribution='uniform',
    random_state=42
)

# Normal distribution
qt_normal = QuantileTransformer(
    n_quantiles=100,
    output_distribution='normal',
    random_state=42
)

print("Uniform:\n", qt_uniform.fit_transform(data))
print("Normal:\n", qt_normal.fit_transform(data))

from sklearn.preprocessing import FunctionTransformer

# Log transformation
log_transformer = FunctionTransformer(
    func=np.log1p,           # log(x + 1)
    inverse_func=np.expm1    # inverse: e^x - 1
)

data = np.array([[0], [1], [10], [100], [1000]])
transformed = log_transformer.fit_transform(data)
print(transformed)
# [[0.   ]
#  [0.693]
#  [2.398]
#  [4.615]
#  [6.909]]

# Koi bhi custom function use kar sakte ho
sqrt_transformer = FunctionTransformer(np.sqrt)

from sklearn.preprocessing import KBinsDiscretizer

data = np.array([[0], [10], [20], [30], [40], [50], [100]])

# Equal-width bins
kbd_width = KBinsDiscretizer(
    n_bins=3,
    encode='ordinal',     # 'onehot', 'onehot-dense', 'ordinal'
    strategy='uniform'    # 'uniform', 'quantile', 'kmeans'
)

print("Uniform:\n", kbd_width.fit_transform(data))

# Equal-frequency bins
kbd_freq = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='quantile')
print("Quantile:\n", kbd_freq.fit_transform(data))

print("Bin edges:", kbd_width.bin_edges_)

from sklearn.preprocessing import PolynomialFeatures

data = np.array([[2, 3]])  # 2 features: a=2, b=3

# Degree 2: 1, a, b, a^2, ab, b^2
poly = PolynomialFeatures(degree=2, include_bias=True)
result = poly.fit_transform(data)
print(result)
# [[ 1.  2.  3.  4.  6.  9.]]

print(poly.get_feature_names_out(['a', 'b']))
# ['1' 'a' 'b' 'a^2' 'a b' 'b^2']

# Interaction only (no powers)
poly_interact = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)
result2 = poly_interact.fit_transform(data)
print(result2)
# [[2. 3. 6.]]  -> a, b, a*b

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.compose import ColumnTransformer
import pandas as pd

# Sample dataset
df = pd.DataFrame({
    'age':    [25, 35, 45, 28, 55],
    'salary': [30000, 60000, 90000, 40000, 120000],
    'city':   ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai', 'Delhi'],
    'gender': ['M', 'F', 'M', 'F', 'M'],
    'churn':  [0, 0, 1, 0, 1]
})

X = df.drop('churn', axis=1)
y = df['churn']

# Numeric aur categorical columns alag karo
numeric_features = ['age', 'salary']
categorical_features = ['city', 'gender']

# Column-specific preprocessors
numeric_transformer = Pipeline(steps=[
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline(steps=[
    ('encoder', OneHotEncoder(drop='first', handle_unknown='ignore'))
])

# ColumnTransformer: different columns pe different transformers
preprocessor = ColumnTransformer(transformers=[
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

# Final pipeline
pipeline = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('classifier', LogisticRegression())
])

# Train/test split ke saath
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

pipeline.fit(X_train, y_train)
print("Accuracy:", pipeline.score(X_test, y_test))

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder

# Complex real-world scenario
preprocessor = ColumnTransformer(
    transformers=[
        ('scale_num', StandardScaler(), ['age', 'income', 'credit_score']),
        ('encode_cat', OneHotEncoder(handle_unknown='ignore'), ['city', 'occupation']),
        ('encode_ord', OrdinalEncoder(categories=[['Low', 'Medium', 'High']]), ['risk_level']),
    ],
    remainder='passthrough'  # Baaki columns as-is rakhna
)

# Feature names extract karna
X_transformed = preprocessor.fit_transform(X_train)
feature_names = preprocessor.get_feature_names_out()
print(feature_names)

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
data = np.array([[1000, 2], [2000, 4], [3000, 6]])

scaled = scaler.fit_transform(data)
print("Scaled:\n", scaled)

# Wapas original values mein convert karo
original = scaler.inverse_transform(scaled)
print("Recovered:\n", original)
# Same as original data

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

# Batch by batch fit karo
for batch in data_batches:  # Generator ya list of chunks
    scaler.partial_fit(batch)

# Ab transform karo
X_scaled = scaler.transform(X_test)

from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline

# ✅ CORRECT: Pipeline ke andar scaler rakhna
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('model', LogisticRegression())
])

scores = cross_val_score(pipe, X, y, cv=5)
# Automatically: train fold pe fit, val fold pe transform


# ❌ WRONG: Pehle scale karna phir CV
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # Data leakage!
scores = cross_val_score(LogisticRegression(), X_scaled, y, cv=5)

from sklearn.base import BaseEstimator, TransformerMixin
import numpy as np

class LogTransformer(BaseEstimator, TransformerMixin):
    """Custom log(1+x) transformer with clipping."""
    
    def __init__(self, clip_min=0.0):
        self.clip_min = clip_min
    
    def fit(self, X, y=None):
        # Koi statistics seekhna ho to yahaan
        self.n_features_in_ = X.shape[1] if hasattr(X, 'shape') else 1
        return self  # Hamesha self return karo
    
    def transform(self, X, y=None):
        X = np.array(X)
        X_clipped = np.clip(X, self.clip_min, None)  # Negative values clip
        return np.log1p(X_clipped)
    
    def get_feature_names_out(self, input_features=None):
        if input_features is None:
            return [f'log_feature_{i}' for i in range(self.n_features_in_)]
        return [f'log_{name}' for name in input_features]

# Use in pipeline
pipe = Pipeline([
    ('log', LogTransformer(clip_min=0)),
    ('scaler', StandardScaler()),
    ('model', LogisticRegression())
])

from sklearn.preprocessing import TargetEncoder

# City column mein 500+ unique values hain
te = TargetEncoder(
    target_type='continuous',  # ya 'binary', 'multiclass'
    smooth='auto',              # Regularization
    cv=5                        # Cross-validation folds
)

X_train_enc = te.fit_transform(X_train[['city']], y_train)
X_test_enc = te.transform(X_test[['city']])

# StandardScaler internally kya karta hai:
class MyStandardScaler:
    def fit(self, X):
        self.mean_ = X.mean(axis=0)     # Column-wise mean
        self.var_  = X.var(axis=0)       # Column-wise variance
        self.scale_ = np.sqrt(self.var_) # Standard deviation
        return self
    
    def transform(self, X):
        return (X - self.mean_) / self.scale_
    
    def inverse_transform(self, X_scaled):
        return X_scaled * self.scale_ + self.mean_

# Degree 2, 3 features se kitne features bante hain?
from sklearn.preprocessing import PolynomialFeatures
import math

def count_poly_features(n_features, degree, include_bias=True):
    from math import comb
    total = sum(comb(n_features + d, d) for d in range(1, degree + 1))
    return total + (1 if include_bias else 0)

# n=10 features, degree=2 → 66 features!
# n=10 features, degree=3 → 286 features!
# n=100 features, degree=2 → 5151 features! (curse of dimensionality)

poly = PolynomialFeatures(degree=2)
print(f"100 features → {poly.fit(np.zeros((1, 100))).n_output_features_} features")

from scipy.sparse import issparse
from sklearn.preprocessing import MaxAbsScaler

# OneHotEncoder returns sparse matrix by default
ohe = OneHotEncoder()  # sparse_output=True by default
X_sparse = ohe.fit_transform(categories_data)

print("Is sparse:", issparse(X_sparse))  # True
print("Memory saved:", f"{X_sparse.nnz / X_sparse.size:.2%}")

# MaxAbsScaler sparse matrices ke saath kaam karta hai
mas = MaxAbsScaler()
X_sparse_scaled = mas.fit_transform(X_sparse)

from sklearn.preprocessing import StandardScaler
import pandas as pd

X = pd.DataFrame({'age': [25, 35, 45], 'salary': [30000, 60000, 90000]})

# DataFrame output chahiye (numpy array nahi)
scaler = StandardScaler()
scaler.set_output(transform='pandas')  # or 'default' for numpy

result = scaler.fit_transform(X)
print(type(result))   # <class 'pandas.core.frame.DataFrame'>
print(result.columns) # ['age', 'salary']

import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV

# Telecom churn dataset simulation
np.random.seed(42)
n = 1000
df = pd.DataFrame({
    'tenure_months':  np.random.randint(1, 72, n),
    'monthly_charges': np.random.uniform(20, 120, n),
    'total_charges':   np.random.uniform(100, 8000, n),
    'contract':        np.random.choice(['Month-to-month', 'One year', 'Two year'], n),
    'payment_method':  np.random.choice(['Credit card', 'Bank transfer', 'Mailed check'], n),
    'internet_service':np.random.choice(['DSL', 'Fiber optic', 'No'], n),
    'tech_support':    np.random.choice(['Yes', 'No'], n),
    'churn':           np.random.choice([0, 1], n, p=[0.73, 0.27])
})

X = df.drop('churn', axis=1)
y = df['churn']

numeric_cols = ['tenure_months', 'monthly_charges', 'total_charges']
ordinal_cols = ['contract']
nominal_cols = ['payment_method', 'internet_service', 'tech_support']

preprocessor = ColumnTransformer([
    ('num', StandardScaler(), numeric_cols),
    ('ord', OrdinalEncoder(categories=[['Month-to-month', 'One year', 'Two year']]), ordinal_cols),
    ('nom', OneHotEncoder(drop='first', handle_unknown='ignore'), nominal_cols),
])

pipeline = Pipeline([
    ('prep', preprocessor),
    ('model', RandomForestClassifier(random_state=42))
])

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

pipeline.fit(X_train, y_train)
print(f"Churn Prediction Accuracy: {pipeline.score(X_test, y_test):.4f}")

from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.linear_model import Ridge

# Product pricing model
# Features: area_sqft, rooms, age, location_score
X = np.array([
    [500, 1, 10, 7],
    [1000, 2, 5, 8],
    [1500, 3, 2, 9],
    [800, 2, 15, 6],
])
y = np.array([25000, 60000, 95000, 40000])

pipeline = Pipeline([
    ('poly', PolynomialFeatures(degree=2, include_bias=False)),
    ('scaler', StandardScaler()),
    ('model', Ridge(alpha=1.0))
])

pipeline.fit(X, y)
new_property = np.array([[1200, 2, 8, 7.5]])
predicted_price = pipeline.predict(new_property)
print(f"Predicted Price: ₹{predicted_price[0]:,.0f}")

from sklearn.preprocessing import LabelEncoder, normalize
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import issparse

texts = [
    "best product ever",
    "terrible quality waste of money",
    "average product nothing special",
    "loved it will buy again",
    "poor customer service",
]
labels = ['positive', 'negative', 'neutral', 'positive', 'negative']

# Label encoding
le = LabelEncoder()
y = le.fit_transform(labels)  # [1, 0, 2, 1, 0]

# TF-IDF + L2 normalization
tfidf = TfidfVectorizer()
X = tfidf.fit_transform(texts)  # Sparse matrix

# L2 normalize rows
from sklearn.preprocessing import normalize
X_normalized = normalize(X, norm='l2')

print("Classes:", le.classes_)
print("Feature shape:", X_normalized.shape)

from sklearn.preprocessing import StandardScaler, KBinsDiscretizer
import pandas as pd
import numpy as np

# Sales data
np.random.seed(42)
dates = pd.date_range('2023-01-01', periods=365)
sales = np.random.exponential(scale=1000, size=365) + \
        500 * np.sin(np.linspace(0, 4*np.pi, 365))  # Seasonal pattern

df = pd.DataFrame({'date': dates, 'sales': sales})
df['day_of_week'] = df['date'].dt.dayofweek
df['month'] = df['date'].dt.month
df['quarter'] = df['date'].dt.quarter

# Sales ko bins mein divide karo (Low/Medium/High)
discretizer = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='quantile')
df['sales_bucket'] = discretizer.fit_transform(df[['sales']])

# Continuous features scale karo
scaler = StandardScaler()
df['sales_scaled'] = scaler.fit_transform(df[['sales']])

print(df[['date', 'sales', 'sales_scaled', 'sales_bucket']].head(10))

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.ensemble import GradientBoostingClassifier

# Titanic data simulate karte hain
titanic = pd.DataFrame({
    'Pclass':   [1, 2, 3, 1, 3, 2, 1, 3],
    'Sex':      ['male', 'female', 'male', 'female', 'male', 'female', 'male', 'female'],
    'Age':      [22, 38, np.nan, 35, 28, np.nan, 54, 27],
    'SibSp':    [1, 1, 0, 1, 0, 0, 0, 0],
    'Fare':     [7.25, 71.28, 7.92, 53.1, 8.05, 13.0, 51.86, 7.92],
    'Embarked': ['S', 'C', 'S', 'S', 'S', 'Q', 'S', None],
    'Survived': [0, 1, 1, 1, 0, 0, 0, 1]
})

X = titanic.drop('Survived', axis=1)
y = titanic['Survived']

# Numeric features: Impute + Scale
numeric_features = ['Age', 'Fare', 'SibSp']
numeric_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median')),  # Missing values
    ('scaler', StandardScaler())                     # Scale
])

# Categorical features: Impute + Encode
categorical_features = ['Sex', 'Embarked']
categorical_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='most_frequent')),  # Missing values
    ('encoder', OneHotEncoder(handle_unknown='ignore'))    # Encode
])

# Ordinal feature (Pclass already numeric but ordinal)
ordinal_features = ['Pclass']
# Already numeric, bas scaler lagado

preprocessor = ColumnTransformer(transformers=[
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features),
    ('ord', StandardScaler(), ordinal_features),
])

# Full pipeline
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('model', GradientBoostingClassifier(n_estimators=100, random_state=42))
])

from sklearn.model_selection import train_test_split, cross_val_score
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)

pipeline.fit(X_train, y_train)

# Cross validation score
cv_scores = cross_val_score(pipeline, X, y, cv=3, scoring='accuracy')

print(f"Train Score: {pipeline.score(X_train, y_train):.4f}")
print(f"Test Score:  {pipeline.score(X_test, y_test):.4f}")
print(f"CV Score:    {cv_scores.mean():.4f} (+/- {cv_scores.std():.4f})")

import numpy as np
import pandas as pd
from sklearn.preprocessing import PowerTransformer, OrdinalEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt

np.random.seed(42)
n = 500

df = pd.DataFrame({
    'experience_years': np.random.randint(0, 20, n),
    'education':        np.random.choice(['High School', 'Bachelors', 'Masters', 'PhD'], n),
    'department':       np.random.choice(['IT', 'Finance', 'HR', 'Marketing'], n),
    # Salary is log-normally distributed (highly skewed) - typical in real world
    'salary': np.random.lognormal(mean=11, sigma=0.5, size=n)
})

print("Salary statistics before transform:")
print(f"  Skewness: {df['salary'].skew():.3f}")

# Target variable bhi transform karo!
from sklearn.preprocessing import PowerTransformer
pt = PowerTransformer(method='yeo-johnson')
y = pt.fit_transform(df[['salary']]).ravel()

print(f"  Skewness after transform: {pd.Series(y).skew():.3f}")

X = df.drop('salary', axis=1)

preprocessor = ColumnTransformer([
    ('scale', PowerTransformer(), ['experience_years']),
    ('ordinal', OrdinalEncoder(categories=[
        [['High School', 'Bachelors', 'Masters', 'PhD']]
    ]), ['education']),
    ('onehot', OneHotEncoder(drop='first', sparse_output=False), ['department']),
])

pipeline = Pipeline([
    ('prep', preprocessor),
    ('model', LinearRegression())
])

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

pipeline.fit(X_train, y_train)
y_pred_transformed = pipeline.predict(X_test)

# Predictions ko original salary mein wapas laao
y_pred_original = pt.inverse_transform(y_pred_transformed.reshape(-1, 1))
y_test_original = pt.inverse_transform(y_test.reshape(-1, 1))

from sklearn.metrics import mean_absolute_error
mae = mean_absolute_error(y_test_original, y_pred_original)
print(f"\nMean Absolute Error: ₹{mae:,.0f}")
print(f"R² Score: {pipeline.score(X_test, y_test):.4f}")

from sklearn.datasets import load_iris
from sklearn.preprocessing import StandardScaler, LabelBinarizer
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split, GridSearchCV

# Iris dataset
iris = load_iris()
X, y = iris.data, iris.target

# Label ko human-readable banao
lb = LabelBinarizer()
y_binary = lb.fit_transform(y)  # [0,1,2] → [[1,0,0],[0,1,0],[0,0,1]]
print("Classes:", lb.classes_)  # [0, 1, 2]

# Pipeline with hyperparameter tuning
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('svm', SVC(probability=True, random_state=42))
])

param_grid = {
    'scaler__with_mean': [True, False],  # Scaler params
    'svm__C': [0.1, 1, 10],             # SVM params
    'svm__kernel': ['rbf', 'linear'],
}

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy', n_jobs=-1)
grid_search.fit(X_train, y_train)

print(f"Best Params: {grid_search.best_params_}")
print(f"Best CV Score: {grid_search.best_score_:.4f}")
print(f"Test Score: {grid_search.score(X_test, y_test):.4f}")

# ❌ WRONG — Data leakage!
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # Puri dataset pe fit kar liya

X_train, X_test = train_test_split(X_scaled, ...)  # Test data leak ho gaya

# ✅ CORRECT
X_train, X_test = train_test_split(X, ...)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)   # Sirf train pe fit
X_test_scaled  = scaler.transform(X_test)          # Test pe sirf transform

# ❌ Error: Unknown category at inference time
ohe = OneHotEncoder()
ohe.fit([['Mumbai'], ['Delhi']])
ohe.transform([['Chennai']])  # ValueError: Found unknown categories

# ✅ FIX: handle_unknown='ignore'
ohe = OneHotEncoder(handle_unknown='ignore')
ohe.fit([['Mumbai'], ['Delhi']])
result = ohe.transform([['Chennai']])  # Silently becomes [0, 0]

# ✅ FIX: handle_unknown='infrequent_if_exist'
ohe = OneHotEncoder(handle_unknown='infrequent_if_exist', min_frequency=2)

# ❌ Error: Box-Cox needs strictly positive values
pt = PowerTransformer(method='box-cox')
data_with_zeros = np.array([[0], [1], [2], [-1]])
pt.fit_transform(data_with_zeros)  # ValueError!

# ✅ FIX 1: Yeo-Johnson use karo
pt = PowerTransformer(method='yeo-johnson')
pt.fit_transform(data_with_zeros)  # Works!

# ✅ FIX 2: Shift data before box-cox
from sklearn.preprocessing import FunctionTransformer
shift_transformer = FunctionTransformer(lambda x: x + abs(x.min()) + 1)

# ❌ Error: Strings in numeric column
data = pd.DataFrame({'salary': ['50000', '60000', 'N/A', '70000']})
scaler = StandardScaler()
scaler.fit_transform(data)  # ValueError: could not convert string to float

# ✅ FIX: Clean data first
data['salary'] = pd.to_numeric(data['salary'], errors='coerce')  # 'N/A' → NaN
# Then use SimpleImputer + StandardScaler in pipeline

# ❌ Common mistake: 1D array dena
data_1d = np.array([1, 2, 3, 4, 5])
scaler = StandardScaler()
scaler.fit_transform(data_1d)  # ValueError: Expected 2D array

# ✅ FIX: Reshape karo
scaler.fit_transform(data_1d.reshape(-1, 1))  # (5,) → (5, 1)

# Ya pandas Series ke liye
scaler.fit_transform(pd.Series([1,2,3]).values.reshape(-1, 1))

# ❌ WRONG: Test set pe fit karna
scaler = StandardScaler()
scaler.fit(X_test)  # Test ke statistics use ho rahe hain!
X_train_scaled = scaler.transform(X_train)

# ✅ CORRECT: Training set pe fit karo
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled  = scaler.transform(X_test)  # Same scaler!

# Scaler ke statistics check karo
scaler = StandardScaler()
scaler.fit(X_train)

print("Mean per feature:", scaler.mean_)
print("Std per feature:", scaler.scale_)
print("Variance:", scaler.var_)
print("N samples seen:", scaler.n_samples_seen_)

# Transform hone ke baad verify karo
X_scaled = scaler.transform(X_train)
print("Scaled mean (should be ~0):", X_scaled.mean(axis=0))
print("Scaled std (should be ~1):",  X_scaled.std(axis=0))

# OneHotEncoder categories check karo
ohe = OneHotEncoder()
ohe.fit(X_cat)
print("Categories:", ohe.categories_)
print("Feature names:", ohe.get_feature_names_out())

# ✅ 1. HAMESHA Pipeline use karo
pipeline = Pipeline([('scaler', StandardScaler()), ('model', LogisticRegression())])

# ✅ 2. Train/test ke baad fit karo
X_train, X_test, y_train, y_test = train_test_split(X, y)
pipeline.fit(X_train, y_train)

# ✅ 3. set_output se DataFrame maintain karo
preprocessor.set_output(transform='pandas')

# ✅ 4. get_params() se hyperparameters check karo
print(scaler.get_params())

# ✅ 5. Preprocessor ko save karo (model ke saath)
import joblib
joblib.dump(pipeline, 'model_pipeline.pkl')
loaded = joblib.load('model_pipeline.pkl')
# Test time pe same transformations apply honge automatically

# Tip 1: Sparse matrix use karo jab possible
ohe = OneHotEncoder()  # sparse_output=True by default — use it!

# Tip 2: n_jobs=-1 se parallel processing
from sklearn.preprocessing import KBinsDiscretizer
kbd = KBinsDiscretizer()  

# Tip 3: Partial fit for big data
for chunk in pd.read_csv('big_data.csv', chunksize=10000):
    scaler.partial_fit(chunk[numeric_cols])

# Tip 4: Memory-efficient: float32 use karo
import numpy as np
X = X.astype(np.float32)

# Tip 5: DataFrame set_output API (sklearn 1.2+)
from sklearn import set_config
set_config(transform_output='pandas')  # Global setting

# ❌ Kabhi bhi untrusted pickle files load mat karo
import pickle
model = pickle.load(open('untrusted_model.pkl', 'rb'))  # Dangerous!

# ✅ joblib use karo + source verify karo
import joblib
model = joblib.load('trusted_model.pkl')

# Production mein: Preprocessing artifacts bhi save karo
artifacts = {
    'preprocessor': preprocessor,
    'model': trained_model,
    'feature_names': list(X.columns),
    'version': '1.0.0'
}
joblib.dump(artifacts, 'production_artifacts.pkl')

# Pattern 1: Constants alag rakhna
NUMERIC_FEATURES = ['age', 'salary', 'tenure']
CATEGORICAL_FEATURES = ['department', 'city']
ORDINAL_FEATURES = ['education']
ORDINAL_CATEGORIES = [['High School', 'Bachelors', 'Masters', 'PhD']]

# Pattern 2: Factory function
def build_preprocessor():
    return ColumnTransformer([
        ('num', Pipeline([
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
        ]), NUMERIC_FEATURES),
        ('cat', Pipeline([
            ('imputer', SimpleImputer(strategy='most_frequent')),
            ('encoder', OneHotEncoder(handle_unknown='ignore'))
        ]), CATEGORICAL_FEATURES),
    ])

# Pattern 3: Feature validation
def validate_features(df, required_features):
    missing = set(required_features) - set(df.columns)
    if missing:
        raise ValueError(f"Missing features: {missing}")

validate_features(X_test, NUMERIC_FEATURES + CATEGORICAL_FEATURES)

import pandas as pd
import numpy as np

def preprocessing_recommendations(df):
    """Data dekhke automatically preprocessing suggest karo."""
    
    print("=" * 60)
    print("PREPROCESSING RECOMMENDATIONS")
    print("=" * 60)
    
    for col in df.columns:
        dtype = df[col].dtype
        nunique = df[col].nunique()
        missing_pct = df[col].isna().mean() * 100
        
        print(f"\n📊 Column: {col}")
        print(f"   Type: {dtype}, Unique: {nunique}, Missing: {missing_pct:.1f}%")
        
        if dtype in ['object', 'category']:
            if nunique == 2:
                print("   ✅ Recommended: LabelEncoder (binary)")
            elif nunique <= 10:
                print("   ✅ Recommended: OneHotEncoder (nominal)")
            elif nunique > 10:
                print("   ✅ Recommended: TargetEncoder (high cardinality)")
        
        elif dtype in ['int64', 'float64']:
            skewness = df[col].skew()
            if abs(skewness) > 1:
                print(f"   📈 High skewness: {skewness:.2f}")
                print("   ✅ Recommended: PowerTransformer or QuantileTransformer")
            else:
                print("   ✅ Recommended: StandardScaler")
        
        if missing_pct > 0:
            print(f"   ⚠️  Missing values: Use SimpleImputer")

# Use karo
preprocessing_recommendations(df)

from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
import pandas as pd

# Pipeline train karo
pipeline.fit(X_train, y_train)

# Feature names extract karo
feature_names = pipeline.named_steps['preprocessor'].get_feature_names_out()

# Feature importance
rf = pipeline.named_steps['model']
importance_df = pd.DataFrame({
    'feature': feature_names,
    'importance': rf.feature_importances_
}).sort_values('importance', ascending=False)

print(importance_df.head(10))

1. Data Collection
    ↓
2. EDA (Exploratory Data Analysis)
   - Distribution check → Scaler choice
   - Missing values check → Imputer strategy
   - Cardinality check → Encoder choice
   - Correlation analysis
    ↓
3. Feature Engineering
   - PolynomialFeatures
   - KBinsDiscretizer
   - FunctionTransformer
    ↓
4. Preprocessing Pipeline Build
   - ColumnTransformer
   - Pipeline
    ↓
5. Train/Test Split (PHIR fit karo!)
    ↓
6. Model Training + Hyperparameter Tuning
    ↓
7. Evaluation
    ↓
8. Save Pipeline (preprocessor + model)
    ↓
9. Production Deployment

import numpy as np
from scipy import stats

def select_scaler(data):
    """Distribution dekhke best scaler recommend karo."""
    
    # Normality test
    _, p_value = stats.normaltest(data)
    skewness = stats.skew(data)
    kurtosis = stats.kurtosis(data)
    
    # Outlier detection
    q1, q3 = np.percentile(data, [25, 75])
    iqr = q3 - q1
    outliers = np.sum((data < q1 - 1.5*iqr) | (data > q3 + 1.5*iqr))
    outlier_pct = outliers / len(data) * 100
    
    print(f"Skewness: {skewness:.3f}")
    print(f"Kurtosis: {kurtosis:.3f}")
    print(f"Outlier %: {outlier_pct:.1f}%")
    print(f"Normality p-value: {p_value:.4f}")
    
    if outlier_pct > 5:
        print("→ Recommendation: RobustScaler")
    elif abs(skewness) > 1:
        print("→ Recommendation: PowerTransformer or QuantileTransformer")
    elif p_value > 0.05:  # Normally distributed
        print("→ Recommendation: StandardScaler")
    else:
        print("→ Recommendation: QuantileTransformer (safe choice)")

# Test
salary_data = np.random.lognormal(10, 1, 1000)
select_scaler(salary_data)

# Correct sequence:
X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.3)
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5)

# Fit only on train!
preprocessor.fit(X_train)
X_train_t = preprocessor.transform(X_train)
X_val_t   = preprocessor.transform(X_val)    # Same scaler!
X_test_t  = preprocessor.transform(X_test)   # Same scaler!

✅ StandardScaler  → Normal data, SVM, LR, Neural Nets
✅ MinMaxScaler    → Image data, KNN, Neural Nets
✅ RobustScaler    → Outlier-heavy data (financial, medical)
✅ MaxAbsScaler    → Sparse data, NLP/TF-IDF
✅ QuantileTransformer → Any distribution, most robust
✅ PowerTransformer → Skewed data, linear model assumptions

✅ LabelEncoder    → Target variable, tree models
✅ OrdinalEncoder  → Ordered categories (size, education)
✅ OneHotEncoder   → Nominal categories, linear/distance models
✅ TargetEncoder   → High-cardinality categories

✅ PolynomialFeatures → Non-linear feature engineering
✅ KBinsDiscretizer   → Continuous → categorical
✅ FunctionTransformer → Custom any function

Data hai → Kya type?

NUMERIC → Outliers hain?
   Yes → RobustScaler
   No  → Skewed?
        Yes → PowerTransformer / QuantileTransformer
        No  → StandardScaler (normal) / MinMaxScaler (neural nets)

CATEGORICAL → Order hai?
   Yes → OrdinalEncoder
   No  → Cardinality?
         High (>50) → TargetEncoder
         Low  (<50) → OneHotEncoder

TARGET VARIABLE → LabelEncoder (multi-class) / as-is (regression)