sklearn.preprocessing Complete Guide: Data Scaling, Encoding & Transformation in Python

Raw Data → Clean Data → Transformed Data → ML Model

1. Feature Scaling      → Numbers ka range equalize karna
2. Encoding             → Strings ko numbers mein convert karna  
3. Normalization        → Rows ko unit norm pe laana
4. Transformation       → Distribution shape change karna
5. Binarization         → Continuous values ko 0/1 mein convert karna
6. Imputation           → Missing values fill karna (SimpleImputer)
7. Polynomial Features  → New features generate karna

# Pura module import karo
from sklearn import preprocessing

# Ya specific classes import karo (recommended)
from sklearn.preprocessing import (
    StandardScaler,
    MinMaxScaler,
    MaxAbsScaler,
    RobustScaler,
    Normalizer,
    Binarizer,
    LabelEncoder,
    OrdinalEncoder,
    OneHotEncoder,
    LabelBinarizer,
    MultiLabelBinarizer,
    PolynomialFeatures,
    FunctionTransformer,
    PowerTransformer,
    QuantileTransformer,
    KBinsDiscretizer,
)

# Step 1: Object banao
scaler = StandardScaler()

# Step 2: Training data pe fit karo (statistics seekhta hai)
scaler.fit(X_train)

# Step 3: Transform karo
X_train_scaled = scaler.transform(X_train)
X_test_scaled  = scaler.transform(X_test)  # Same scaler use karo!

# Ya fit + transform ek saath (sirf training data pe)
X_train_scaled = scaler.fit_transform(X_train)

from sklearn.preprocessing import StandardScaler
import numpy as np

data = np.array([[1000, 2], [2000, 4], [3000, 6], [4000, 8]])

scaler = StandardScaler()
scaled = scaler.fit_transform(data)

print("Original:\n", data)
print("\nScaled:\n", scaled)
print("\nMean:", scaler.mean_)
print("Std:", scaler.scale_)

Original:
 [[1000    2]
  [2000    4]
  [3000    6]
  [4000    8]]

Scaled:
 [[-1.34164079 -1.34164079]
  [-0.4472136  -0.4472136 ]
  [ 0.4472136   0.4472136 ]
  [ 1.34164079  1.34164079]]

Mean: [2500.    5.]
Std:  [1118.03398875    2.23606798]

from sklearn.preprocessing import MinMaxScaler

data = np.array([[100], [200], [300], [400], [500]])

scaler = MinMaxScaler(feature_range=(0, 1))  # Default
scaled = scaler.fit_transform(data)
print(scaled)
# [[0.  ], [0.25], [0.5 ], [0.75], [1.  ]]

# Custom range: -1 to 1
scaler2 = MinMaxScaler(feature_range=(-1, 1))
scaled2 = scaler2.fit_transform(data)
print(scaled2)
# [[-1. ], [-0.5], [0.  ], [0.5 ], [1.  ]]

from sklearn.preprocessing import MaxAbsScaler

data = np.array([[-10], [-5], [0], [5], [10]])

scaler = MaxAbsScaler()
scaled = scaler.fit_transform(data)
print(scaled)
# [[-1. ], [-0.5], [0. ], [0.5], [1. ]]

from sklearn.preprocessing import RobustScaler

# Outliers ke saath data
data = np.array([[1], [2], [3], [4], [100]])  # 100 outlier hai

scaler = RobustScaler()
scaled = scaler.fit_transform(data)
print(scaled)
# [[-1.  ], [-0.5 ], [0.   ], [0.5  ], [48.5 ]]

print("Median:", scaler.center_)   # [3.]
print("IQR:", scaler.scale_)       # [2.]

from sklearn.preprocessing import Normalizer

data = np.array([[3, 4], [1, 2], [6, 8]])

# L2 norm (default): sqrt(sum of squares) = 1
norm_l2 = Normalizer(norm='l2')
print(norm_l2.fit_transform(data))
# [[0.6, 0.8], [0.447, 0.894], [0.6, 0.8]]

# L1 norm: sum of absolute values = 1
norm_l1 = Normalizer(norm='l1')
print(norm_l1.fit_transform(data))
# [[0.429, 0.571], [0.333, 0.667], [0.429, 0.571]]

# Max norm: max value = 1
norm_max = Normalizer(norm='max')
print(norm_max.fit_transform(data))
# [[0.75, 1.0], [0.5, 1.0], [0.75, 1.0]]

from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()

# String labels
cities = ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai', 'Delhi']
encoded = le.fit_transform(cities)
print(encoded)       # [2 1 0 2 1]
print(le.classes_)   # ['Bangalore' 'Delhi' 'Mumbai']

# Inverse transform
print(le.inverse_transform([2, 1, 0]))  # ['Mumbai' 'Delhi' 'Bangalore']

from sklearn.preprocessing import OrdinalEncoder

data = np.array([
    ['Low', 'Small'],
    ['Medium', 'Medium'],
    ['High', 'Large'],
    ['Low', 'Large']
])

encoder = OrdinalEncoder(
    categories=[
        ['Low', 'Medium', 'High'],    # Column 1 order
        ['Small', 'Medium', 'Large']   # Column 2 order
    ]
)

encoded = encoder.fit_transform(data)
print(encoded)
# [[0. 0.]
#  [1. 1.]
#  [2. 2.]
#  [0. 2.]]

from sklearn.preprocessing import OneHotEncoder
import pandas as pd

data = np.array([['Mumbai'], ['Delhi'], ['Bangalore'], ['Mumbai']])

ohe = OneHotEncoder(
    sparse_output=False,    # numpy array return karo
    drop='first',           # Dummy variable trap avoid karo
    handle_unknown='ignore' # New categories ko ignore karo
)

encoded = ohe.fit_transform(data)
print(encoded)
# [[0. 1.]    <- Mumbai
#  [1. 0.]    <- Delhi  
#  [0. 0.]    <- Bangalore (dropped - first)
#  [0. 1.]]   <- Mumbai

print(ohe.categories_)
print(ohe.get_feature_names_out(['city']))
# ['city_Delhi' 'city_Mumbai']

from sklearn.preprocessing import LabelBinarizer

lb = LabelBinarizer()
cities = ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai']

encoded = lb.fit_transform(cities)
print(encoded)
# [[0 0 1]   <- Mumbai
#  [0 1 0]   <- Delhi
#  [1 0 0]   <- Bangalore
#  [0 0 1]]  <- Mumbai

print(lb.classes_)  # ['Bangalore' 'Delhi' 'Mumbai']

from sklearn.preprocessing import MultiLabelBinarizer

mlb = MultiLabelBinarizer()

# Ek user multiple genres pasand karta hai
genres = [
    ['Action', 'Drama'],
    ['Comedy'],
    ['Action', 'Comedy', 'Romance'],
    ['Drama', 'Romance']
]

encoded = mlb.fit_transform(genres)
print(mlb.classes_)  # ['Action' 'Comedy' 'Drama' 'Romance']
print(encoded)
# [[1 0 1 0]
#  [0 1 0 0]
#  [1 1 0 1]
#  [0 0 1 1]]

from sklearn.preprocessing import Binarizer

data = np.array([[0.1, 0.5, 0.9], [0.2, 0.6, 0.4]])

# Default threshold = 0.0
binarizer = Binarizer(threshold=0.5)
result = binarizer.fit_transform(data)
print(result)
# [[0 0 1]
#  [0 1 0]]

from sklearn.preprocessing import PowerTransformer

data = np.array([[1], [2], [3], [4], [100], [1000]])  # Skewed data

# Yeo-Johnson: positive aur negative dono values handle karta hai
pt_yj = PowerTransformer(method='yeo-johnson')
transformed = pt_yj.fit_transform(data)
print("Yeo-Johnson:\n", transformed)

# Box-Cox: sirf positive values
pt_bc = PowerTransformer(method='box-cox')
transformed_bc = pt_bc.fit_transform(data)
print("Box-Cox lambda:", pt_bc.lambdas_)

from sklearn.preprocessing import QuantileTransformer

data = np.array([[1], [2], [3], [100], [1000], [50000]])

# Uniform distribution (0 to 1)
qt_uniform = QuantileTransformer(
    n_quantiles=100,
    output_distribution='uniform',
    random_state=42
)

# Normal distribution
qt_normal = QuantileTransformer(
    n_quantiles=100,
    output_distribution='normal',
    random_state=42
)

print("Uniform:\n", qt_uniform.fit_transform(data))
print("Normal:\n", qt_normal.fit_transform(data))

from sklearn.preprocessing import FunctionTransformer

# Log transformation
log_transformer = FunctionTransformer(
    func=np.log1p,           # log(x + 1)
    inverse_func=np.expm1    # inverse: e^x - 1
)

data = np.array([[0], [1], [10], [100], [1000]])
transformed = log_transformer.fit_transform(data)
print(transformed)
# [[0.   ]
#  [0.693]
#  [2.398]
#  [4.615]
#  [6.909]]

# Koi bhi custom function use kar sakte ho
sqrt_transformer = FunctionTransformer(np.sqrt)

from sklearn.preprocessing import KBinsDiscretizer

data = np.array([[0], [10], [20], [30], [40], [50], [100]])

# Equal-width bins
kbd_width = KBinsDiscretizer(
    n_bins=3,
    encode='ordinal',     # 'onehot', 'onehot-dense', 'ordinal'
    strategy='uniform'    # 'uniform', 'quantile', 'kmeans'
)

print("Uniform:\n", kbd_width.fit_transform(data))

# Equal-frequency bins
kbd_freq = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='quantile')
print("Quantile:\n", kbd_freq.fit_transform(data))

print("Bin edges:", kbd_width.bin_edges_)

from sklearn.preprocessing import PolynomialFeatures

data = np.array([[2, 3]])  # 2 features: a=2, b=3

# Degree 2: 1, a, b, a^2, ab, b^2
poly = PolynomialFeatures(degree=2, include_bias=True)
result = poly.fit_transform(data)
print(result)
# [[ 1.  2.  3.  4.  6.  9.]]

print(poly.get_feature_names_out(['a', 'b']))
# ['1' 'a' 'b' 'a^2' 'a b' 'b^2']

# Interaction only (no powers)
poly_interact = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)
result2 = poly_interact.fit_transform(data)
print(result2)
# [[2. 3. 6.]]  -> a, b, a*b

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.compose import ColumnTransformer
import pandas as pd

# Sample dataset
df = pd.DataFrame({
    'age':    [25, 35, 45, 28, 55],
    'salary': [30000, 60000, 90000, 40000, 120000],
    'city':   ['Mumbai', 'Delhi', 'Bangalore', 'Mumbai', 'Delhi'],
    'gender': ['M', 'F', 'M', 'F', 'M'],
    'churn':  [0, 0, 1, 0, 1]
})

X = df.drop('churn', axis=1)
y = df['churn']

# Numeric aur categorical columns alag karo
numeric_features = ['age', 'salary']
categorical_features = ['city', 'gender']

# Column-specific preprocessors
numeric_transformer = Pipeline(steps=[
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline(steps=[
    ('encoder', OneHotEncoder(drop='first', handle_unknown='ignore'))
])

# ColumnTransformer: different columns pe different transformers
preprocessor = ColumnTransformer(transformers=[
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

# Final pipeline
pipeline = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('classifier', LogisticRegression())
])

# Train/test split ke saath
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

pipeline.fit(X_train, y_train)
print("Accuracy:", pipeline.score(X_test, y_test))

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder

# Complex real-world scenario
preprocessor = ColumnTransformer(
    transformers=[
        ('scale_num', StandardScaler(), ['age', 'income', 'credit_score']),
        ('encode_cat', OneHotEncoder(handle_unknown='ignore'), ['city', 'occupation']),
        ('encode_ord', OrdinalEncoder(categories=[['Low', 'Medium', 'High']]), ['risk_level']),
    ],
    remainder='passthrough'  # Baaki columns as-is rakhna
)

# Feature names extract karna
X_transformed = preprocessor.fit_transform(X_train)
feature_names = preprocessor.get_feature_names_out()
print(feature_names)

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
data = np.array([[1000, 2], [2000, 4], [3000, 6]])

scaled = scaler.fit_transform(data)
print("Scaled:\n", scaled)

# Wapas original values mein convert karo
original = scaler.inverse_transform(scaled)
print("Recovered:\n", original)
# Same as original data

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

# Batch by batch fit karo
for batch in data_batches:  # Generator ya list of chunks
    scaler.partial_fit(batch)

# Ab transform karo
X_scaled = scaler.transform(X_test)

from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline

# ✅ CORRECT: Pipeline ke andar scaler rakhna
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('model', LogisticRegression())
])

scores = cross_val_score(pipe, X, y, cv=5)
# Automatically: train fold pe fit, val fold pe transform


# ❌ WRONG: Pehle scale karna phir CV
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)  # Data leakage!
scores = cross_val_score(LogisticRegression(), X_scaled, y, cv=5)

from sklearn.base import BaseEstimator, TransformerMixin
import numpy as np

class LogTransformer(BaseEstimator, TransformerMixin):
    """Custom log(1+x) transformer with clipping."""
    
    def __init__(self, clip_min=0.0):
        self.clip_min = clip_min
    
    def fit(self, X, y=None):
        # Koi statistics seekhna ho to yahaan
        self.n_features_in_ = X.shape[1] if hasattr(X, 'shape') else 1
        return self  # Hamesha self return karo
    
    def transform(self, X, y=None):
        X = np.array(X)
        X_clipped = np.clip(X, self.clip_min, None)  # Negative values clip
        return np.log1p(X_clipped)
    
    def get_feature_names_out(self, input_features=None):
        if input_features is None:
            return [f'log_feature_{i}' for i in range(self.n_features_in_)]
        return [f'log_{name}' for name in input_features]

# Use in pipeline
pipe = Pipeline([
    ('log', LogTransformer(clip_min=0)),
    ('scaler', StandardScaler()),
    ('model', LogisticRegression())
])

from sklearn.preprocessing import TargetEncoder

# City column mein 500+ unique values hain
te = TargetEncoder(
    target_type='continuous',  # ya 'binary', 'multiclass'
    smooth='auto',              # Regularization
    cv=5                        # Cross-validation folds
)

X_train_enc = te.fit_transform(X_train[['city']], y_train)
X_test_enc = te.transform(X_test[['city']])

# StandardScaler internally kya karta hai:
class MyStandardScaler:
    def fit(self, X):
        self.mean_ = X.mean(axis=0)     # Column-wise mean
        self.var_  = X.var(axis=0)       # Column-wise variance
        self.scale_ = np.sqrt(self.var_) # Standard deviation
        return self
    
    def transform(self, X):
        return (X - self.mean_) / self.scale_
    
    def inverse_transform(self, X_scaled):
        return X_scaled * self.scale_ + self.mean_

# Degree 2, 3 features se kitne features bante hain?
from sklearn.preprocessing import PolynomialFeatures
import math

def count_poly_features(n_features, degree, include_bias=True):
    from math import comb
    total = sum(comb(n_features + d, d) for d in range(1, degree + 1))
    return total + (1 if include_bias else 0)

# n=10 features, degree=2 → 66 features!
# n=10 features, degree=3 → 286 features!
# n=100 features, degree=2 → 5151 features! (curse of dimensionality)

poly = PolynomialFeatures(degree=2)
print(f"100 features → {poly.fit(np.zeros((1, 100))).n_output_features_} features")

from scipy.sparse import issparse
from sklearn.preprocessing import MaxAbsScaler

# OneHotEncoder returns sparse matrix by default
ohe = OneHotEncoder()  # sparse_output=True by default
X_sparse = ohe.fit_transform(categories_data)

print("Is sparse:", issparse(X_sparse))  # True
print("Memory saved:", f"{X_sparse.nnz / X_sparse.size:.2%}")

# MaxAbsScaler sparse matrices ke saath kaam karta hai
mas = MaxAbsScaler()
X_sparse_scaled = mas.fit_transform(X_sparse)

from sklearn.preprocessing import StandardScaler
import pandas as pd

X = pd.DataFrame({'age': [25, 35, 45], 'salary': [30000, 60000, 90000]})

# DataFrame output chahiye (numpy array nahi)
scaler = StandardScaler()
scaler.set_output(transform='pandas')  # or 'default' for numpy

result = scaler.fit_transform(X)
print(type(result))   # <class 'pandas.core.frame.DataFrame'>
print(result.columns) # ['age', 'salary']

import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV

# Telecom churn dataset simulation
np.random.seed(42)
n = 1000
df = pd.DataFrame({
    'tenure_months':  np.random.randint(1, 72, n),
    'monthly_charges': np.random.uniform(20, 120, n),
    'total_charges':   np.random.uniform(100, 8000, n),
    'contract':        np.random.choice(['Month-to-month', 'One year', 'Two year'], n),
    'payment_method':  np.random.choice(['Credit card', 'Bank transfer', 'Mailed check'], n),
    'internet_service':np.random.choice(['DSL', 'Fiber optic', 'No'], n),
    'tech_support':    np.random.choice(['Yes', 'No'], n),
    'churn':           np.random.choice([0, 1], n, p=[0.73, 0.27])
})

X = df.drop('churn', axis=1)
y = df['churn']

numeric_cols = ['tenure_months', 'monthly_charges', 'total_charges']
ordinal_cols = ['contract']
nominal_cols = ['payment_method', 'internet_service', 'tech_support']

preprocessor = ColumnTransformer([
    ('num', StandardScaler(), numeric_cols),
    ('ord', OrdinalEncoder(categories=[['Month-to-month', 'One year', 'Two year']]), ordinal_cols),
    ('nom', OneHotEncoder(drop='first', handle_unknown='ignore'), nominal_cols),
])

pipeline = Pipeline([
    ('prep', preprocessor),
    ('model', RandomForestClassifier(random_state=42))
])

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

pipeline.fit(X_train, y_train)
print(f"Churn Prediction Accuracy: {pipeline.score(X_test, y_test):.4f}")

from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.linear_model import Ridge

# Product pricing model
# Features: area_sqft, rooms, age, location_score
X = np.array([
    [500, 1, 10, 7],
    [1000, 2, 5, 8],
    [1500, 3, 2, 9],
    [800, 2, 15, 6],
])
y = np.array([25000, 60000, 95000, 40000])

pipeline = Pipeline([
    ('poly', PolynomialFeatures(degree=2, include_bias=False)),
    ('scaler', StandardScaler()),
    ('model', Ridge(alpha=1.0))
])

pipeline.fit(X, y)
new_property = np.array([[1200, 2, 8, 7.5]])
predicted_price = pipeline.predict(new_property)
print(f"Predicted Price: ₹{predicted_price[0]:,.0f}")

from sklearn.preprocessing import LabelEncoder, normalize
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import issparse

texts = [
    "best product ever",
    "terrible quality waste of money",
    "average product nothing special",
    "loved it will buy again",
    "poor customer service",
]
labels = ['positive', 'negative', 'neutral', 'positive', 'negative']

# Label encoding
le = LabelEncoder()
y = le.fit_transform(labels)  # [1, 0, 2, 1, 0]

# TF-IDF + L2 normalization
tfidf = TfidfVectorizer()
X = tfidf.fit_transform(texts)  # Sparse matrix

# L2 normalize rows
from sklearn.preprocessing import normalize
X_normalized = normalize(X, norm='l2')

print("Classes:", le.classes_)
print("Feature shape:", X_normalized.shape)

from sklearn.preprocessing import StandardScaler, KBinsDiscretizer
import pandas as pd
import numpy as np

# Sales data
np.random.seed(42)
dates = pd.date_range('2023-01-01', periods=365)
sales = np.random.exponential(scale=1000, size=365) + \
        500 * np.sin(np.linspace(0, 4*np.pi, 365))  # Seasonal pattern

df = pd.DataFrame({'date': dates, 'sales': sales})
df['day_of_week'] = df['date'].dt.dayofweek
df['month'] = df['date'].dt.month
df['quarter'] = df['date'].dt.quarter

# Sales ko bins mein divide karo (Low/Medium/High)
discretizer = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='quantile')
df['sales_bucket'] = discretizer.fit_transform(df[['sales']])

# Continuous features scale karo
scaler = StandardScaler()
df['sales_scaled'] = scaler.fit_transform(df[['sales']])

print(df[['date', 'sales', 'sales_scaled', 'sales_bucket']].head(10))