Pandas / Polars

# pip
pip install polars

# conda
conda install polars

# With all optional features
pip install polars[all]

# Specific extras
pip install polars[numpy,pandas,pyarrow]

import polars as pl

# Version check
print(pl.__version__)

# Optional: expression namespace
from polars import col, lit, when

# From dict
df = pl.DataFrame({
    "name": ["Alice", "Bob", "Carol"],
    "age": [30, 25, 35],
    "score": [85.5, 92.0, 78.3]
})

# No index concept — use row_nr if needed
df = pl.DataFrame({
    "x": [1,2,3]
}).with_row_index("idx")

# From list of dicts
df = pl.DataFrame([
    {"a": 1, "b": 2},
    {"a": 3, "b": 4}
])

# With explicit schema
df = pl.DataFrame(
    {"x": [1,2], "y": [3.0,4.0]},
    schema={"x": pl.Int32, "y": pl.Float64}
)

# Create a Series
s = pl.Series("values", [1, 2, 3])

# Access by position (no label index)
s[0]            # integer indexing
s.gather([0, 2]) # multiple indices

# String methods (via expressions)
df.select(pl.col("name").str.to_uppercase())
df.select(pl.col("name").str.contains("li"))

# Vectorized math
df.select((pl.col("score") * 1.1).round(1))

# Select columns
df.select("name", "age")
df.select(pl.col("name", "age"))

# Add / mutate column (returns new df)
df = df.with_columns(
    (pl.col("age") >= 30).alias("senior")
)

# Multiple new columns at once
df = df.with_columns(
    (pl.col("age") >= 30).alias("senior"),
    pl.when(pl.col("score") >= 90)
      .then(pl.lit("A"))
      .otherwise(pl.lit("B"))
      .alias("grade")
)

# Drop columns
df.drop("score")

# Rename
df.rename({"age": "years"})

# Read (eager)
df = pl.read_csv("data.csv")
df = pl.read_csv("data.csv",
    columns=["a", "b"],
    dtypes={"a": pl.Int32},
    n_rows=1000,
    try_parse_dates=True
)

# Lazy scan (larger than memory)
lf = pl.scan_csv("big.csv")
result = lf.filter(
    pl.col("a") > 100
).collect()

# Write
df.write_csv("out.csv")

# Native Rust parquet reader
df = pl.read_parquet("data.parquet")
df = pl.read_parquet("data.parquet",
    columns=["a", "b"]
)

# Write with compression
df.write_parquet("out.parquet",
    compression="zstd"
)

# Lazy scan — predicate pushdown!
lf = pl.scan_parquet("s3://bucket/data.parquet")
result = lf.filter(
    pl.col("date") > "2024-01-01"
).select("a", "b").collect()

# Expression-based filter
df.filter(pl.col("age") > 25)

# Multiple conditions
df.filter(
    (pl.col("age") > 25) & (pl.col("score") >= 80)
)

# Using SQL (optional)
pl.sql("SELECT * FROM df WHERE age > 25")

# is_in
df.filter(pl.col("name").is_in(["Alice", "Bob"]))

# String contains
df.filter(pl.col("name").str.starts_with("A"))

# is_between
df.filter(pl.col("age").is_between(25, 35))

# Null filtering
df.filter(pl.col("score").is_not_null())

# Head / tail / sample
df.head(5)
df.tail(5)
df.sample(3)

# No label-based index; use row/item
df.row(0)                    # tuple
df.item(0, "name")           # single cell

# Slicing
df.slice(0, 5)               # offset, length
df[0:5]                      # Python slice syntax

# Polars is immutable — use with_columns
df = df.with_columns(
    pl.when(pl.col("age") > 30)
      .then(pl.lit(True))
      .otherwise(pl.lit(False))
      .alias("senior")
)

# Gather specific rows
df.gather([0, 3, 7])

# Single aggregation
df.group_by("dept").agg(
    pl.col("salary").mean()
)

# Multiple aggregations
df.group_by("dept").agg(
    pl.col("salary").mean().alias("avg_salary"),
    pl.col("salary").max().alias("max_salary"),
    pl.col("age").count().alias("headcount"),
)

# Expressions are composable
df.group_by("dept").agg(
    (pl.col("salary").max() - pl.col("salary").min())
      .alias("salary_range")
)

# Group by multiple columns
df.group_by("dept", "level").agg(
    pl.all().mean()
)

# Over expressions (like SQL window funcs)
df = df.with_columns(
    ((pl.col("salary") - pl.col("salary").mean().over("dept"))
     / pl.col("salary").std().over("dept"))
    .alias("z_score")
)

# Filter groups
df.filter(
    pl.col("salary").mean().over("dept") > 50000
)

# Top N per group
df.sort("salary", descending=True) \
  .group_by("dept").head(3)

# Pivot
df.pivot(
    on="level",
    index="dept",
    values="salary",
    aggregate_function="mean"
)

# Inner join
result = orders.join(customers,
    on="customer_id",
    how="inner"
)

# Left join
result = orders.join(customers,
    on="customer_id",
    how="left"
)

# Join on different column names
result = orders.join(customers,
    left_on="cust_id",
    right_on="id"
)

# Multi-key join
result = df1.join(df2,
    on=["year", "month"]
)

# Vertical stack (union)
result = pl.concat([df1, df2, df3])

# Horizontal (side by side)
result = pl.concat([df1, df2], how="horizontal")

# Diagonal (union with different schemas)
result = pl.concat([df1, df2],
    how="diagonal"
)

# Running total per group
df = df.with_columns(
    pl.col("sales").cum_sum().over("dept")
      .alias("cum_sales")
)

# Rank within group
df = df.with_columns(
    pl.col("sales").rank(descending=True,
        method="dense").over("dept")
      .alias("rank")
)

# Shift (lag/lead)
df = df.with_columns(
    pl.col("sales").shift(1).over("dept")
      .alias("prev_sales")
)

# Pct change within group
df = df.with_columns(
    pl.col("sales").pct_change().over("dept")
      .alias("growth")
)

# Rolling average
df = df.with_columns(
    pl.col("price").rolling_mean(window_size=7)
      .alias("ma7")
)

# Rolling std with min_periods
df = df.with_columns(
    pl.col("price").rolling_std(
        window_size=20, min_periods=10
    ).alias("vol")
)

# Expanding (cumulative) window
df = df.with_columns(
    pl.col("price").cum_max().alias("cum_max")
)

# EWM (exponential weighted)
df = df.with_columns(
    pl.col("price").ewm_mean(span=12)
      .alias("ewm")
)