import os
import platform
from dataclasses import dataclass
from pathlib import Path

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn
import xgboost
from IPython.display import display

pd.set_option("display.width", 160)
pd.set_option("display.max_columns", 40)

TARGET_YEAR = 2026                          # the year to forecast
LAST_OBSERVED_YEAR = TARGET_YEAR - 1        # the extract stops at 2025-12-31
REQUIRED_BACKTEST_YEARS = (2023, 2024, 2025)  # required by the brief
PERCENT = 100.0                             # fraction -> percentage
DAYS_PER_YEAR = 365.25                      # annualisation of intervals between leases

# The original CSV files go next to the notebook, as for the starter.
# Optional environment variable if you prefer to keep them elsewhere.
DATA_DIR = Path(os.environ.get("EQUINOXE_DATA_DIR", "."))
SOURCE_FILES = {"listings": "equinoxe_listings.csv", "leases": "equinoxe_lease_history.csv",
                "concessions": "equinoxe_concessions.csv", "asking": "equinoxe_asking_history.csv"}
missing_files = [file_name for file_name in SOURCE_FILES.values() if not (DATA_DIR / file_name).is_file()]
if missing_files:
    raise FileNotFoundError(f"Place the original CSV files next to the notebook (or set EQUINOXE_DATA_DIR). Missing: {missing_files}")

source_tables = {table: pd.read_csv(DATA_DIR / file_name) for table, file_name in SOURCE_FILES.items()}
listings, leases = source_tables["listings"], source_tables["leases"]
concessions, asking = source_tables["concessions"], source_tables["asking"]

for name, table in source_tables.items():
    print(f"{name:<12} {len(table):>6,} rows   {table.shape[1]:>2} columns")
print({"python": platform.python_version(), "pandas": pd.__version__, "numpy": np.__version__,
       "matplotlib": matplotlib.__version__, "scikit-learn": sklearn.__version__, "xgboost": xgboost.__version__})

listings      1,061 rows   29 columns
leases        4,302 rows   35 columns
concessions   3,560 rows   18 columns
asking        3,857 rows   14 columns
{'python': '3.13.16', 'pandas': '2.2.3', 'numpy': '2.3.5', 'matplotlib': '3.11.2', 'scikit-learn': '1.9.1', 'xgboost': '2.1.4'}

LEASE_DATE_COLUMNS = ["sLeaseFrom", "sLeaseTo", "sSignDate", "sAvailable"]
for column in LEASE_DATE_COLUMNS:
    if column in leases.columns:
        leases[column] = pd.to_datetime(leases[column], errors="coerce")

leases["year"] = leases.sLeaseFrom.dt.year
asking["date"] = pd.to_datetime(asking.sMonth + "-01")

lease_column_profile = pd.DataFrame({"type": leases.dtypes.astype(str),
                                     "missing_values": leases.isna().sum(),
                                     "distinct_values": leases.nunique()})
lease_column_profile

UNIT_KEY = ["sPropCode", "sUnitCode"]       # safe unit key (never sSite + sUnitCode)
PROMO_CODE = "PromoPay"

COLUMN_RENAMES = {"sPropCode": "property_code", "sUnitCode": "unit_code", "sState": "province", "sBeds": "bedrooms",
                  "sSignDate": "sign_date", "sLeaseFrom": "lease_start", "sLeaseTo": "lease_end", "sTermSeq": "term_seq",
                  "sRent": "face_rent", "sRentEffective": "effective_rent"}


def normalise_unit_key(table):
    # Copy with a cleaned unit key: whitespace removed, property code in lower case.
    keyed = table.copy()
    keyed["sPropCode"] = keyed["sPropCode"].astype(str).str.strip().str.lower()
    keyed["sUnitCode"] = keyed["sUnitCode"].astype(str).str.strip()
    return keyed


promo_entries = normalise_unit_key(concessions[concessions.sChargeCode.eq(PROMO_CODE)])
unit_listed_rent = normalise_unit_key(listings)[UNIT_KEY + ["sRent"]]
promo_entries = promo_entries.merge(unit_listed_rent, on=UNIT_KEY, how="left", validate="many_to_one")
promo_check = pd.Series({
    "PromoPay entries": len(promo_entries),
    "share with sMonths = 1": (promo_entries.sMonths == 1).mean(),
    "share with negative sAmount": (promo_entries.sAmount < 0).mean(),
    "median credit / unit rent": (-promo_entries.sAmount / promo_entries.sRent).median(),
})
display(promo_check.round(3))

COLUMN_MEANINGS = {
    "sPropCode": "property code (one phase of a building); half of the unit key",
    "sUnitCode": "unit number within the property; the other half of the unit key",
    "sState": "province: Quebec (TAL) or Ontario (guideline)",
    "sBeds": "number of bedrooms",
    "sSignDate": "signature date: used to know what was known at a given date",
    "sLeaseFrom": "lease start: date of the measured event",
    "sLeaseTo": "contractual lease end: used to anticipate the leases expiring in 2026",
    "sTermSeq": "rank of the lease in the unit's history: defines the previous lease",
    "sRent": "monthly contractual (face) rent, the rent written in the lease",
    "sRentEffective": "monthly effective rent, after concessions amortised over the lease: what is collected",
}
column_dictionary = pd.DataFrame({"new name": COLUMN_RENAMES, "what the column represents": COLUMN_MEANINGS})
column_dictionary.index.name = "original column"
column_dictionary

PromoPay entries               1494.000
share with sMonths = 1            0.882
share with negative sAmount       1.000
median credit / unit rent         0.711
dtype: float64

REQUIRED_LEASE_COLUMNS = {"sPropCode", "sUnitCode", "sState", "sBeds", "sRent", "sRentEffective",
                          "sSignDate", "sLeaseFrom", "sLeaseTo", "sTermSeq", "sRenewal"}
KNOWN_PROVINCES = ["Quebec", "Ontario"]
LEASE_NUMERIC_COLUMNS = ["sRent", "sRentEffective", "sTermSeq", "sRenewal", "sBeds"]


class DataContractError(ValueError):
    # An input does not meet an assumption the model depends on.
    pass


def validate_leases(lease_table):
    # Validated copy of the leases: dates and numbers converted, key cleaned, rules checked.
    if not isinstance(lease_table, pd.DataFrame) or lease_table.empty:
        raise DataContractError("leases must be a non-empty DataFrame of lease history")
    missing_columns = REQUIRED_LEASE_COLUMNS - set(lease_table.columns)
    if missing_columns:
        raise DataContractError(f"missing columns: {sorted(missing_columns)}")
    checked = lease_table.copy()
    for column in ["sSignDate", "sLeaseFrom", "sLeaseTo"]:
        checked[column] = pd.to_datetime(checked[column], errors="coerce")
    for column in LEASE_NUMERIC_COLUMNS:
        checked[column] = pd.to_numeric(checked[column], errors="coerce")
    if checked[list(REQUIRED_LEASE_COLUMNS)].isna().any().any():
        raise DataContractError("missing or unreadable values in the required columns")
    checked = normalise_unit_key(checked)
    if checked[UNIT_KEY].eq("").any().any() or not checked["sState"].isin(KNOWN_PROVINCES).all():
        raise DataContractError("empty unit key or unknown province")
    if not np.isfinite(checked[LEASE_NUMERIC_COLUMNS].to_numpy()).all():
        raise DataContractError("non-finite numeric value")
    if not checked["sRenewal"].isin([0, 1]).all():
        raise DataContractError("sRenewal must be 0 or 1")
    if ((checked["sTermSeq"] < 1) | (checked["sTermSeq"] % 1 != 0)).any():
        raise DataContractError("sTermSeq must be a positive integer")
    if checked.duplicated(UNIT_KEY + ["sTermSeq"]).any():
        raise DataContractError("duplicated lease sequence for the same unit")
    if (checked[["sRent", "sRentEffective"]] <= 0).any().any():
        raise DataContractError("rents must be positive")
    if (checked["sRentEffective"] > checked["sRent"]).any():
        raise DataContractError("effective rent greater than contractual (face) rent")
    if ((checked["sLeaseTo"] < checked["sLeaseFrom"]) | (checked["sSignDate"] > checked["sLeaseFrom"])).any():
        raise DataContractError("inconsistent lease or signature dates")
    if checked.groupby(UNIT_KEY)["sState"].nunique().gt(1).any():
        raise DataContractError("a unit cannot change province")
    return checked


validated_leases = validate_leases(leases)
print(f"Data contract met: {len(validated_leases):,} leases, {validated_leases.groupby(UNIT_KEY).ngroups:,} units.")

Data contract met: 4,302 leases, 1,061 units.

print(listings.groupby(["sState", "sCity", "sBuilding"]).size().to_string())
print()
print(listings.groupby(["sBeds", "sBaths"])
      .agg(units=("sRent", "size"),
           median_rent=("sRent", "median"),
           median_sqft=("sSqft", "median"))
      .to_string())

sState   sCity          sBuilding     
Ontario  Ottawa         The Met           124
Quebec   Laval          Daniel-Johnson    134
                        Levesque           84
                        Saint-Elzear      361
         Mont-Royal     Le Carlyle        192
         Pointe-Claire  Westpark          166

              units  median_rent  median_sqft
sBeds sBaths                                 
0     1.0        49       2565.0        550.0
1     1.0       350       1380.0        600.0
2     1.0       358       2440.0       1150.0
      2.0       281       2605.0       1265.0
3     1.0         2       2687.5       1597.5
      2.0        20       3237.5       1545.0
      3.0         1       4115.0       1905.0

FIRST_YEAR_SHOWN = 2019

naive = (leases[leases.year.between(FIRST_YEAR_SHOWN, LAST_OBSERVED_YEAR)]
         .groupby("year")
         .agg(leases=("sRent", "size"), median_rent=("sRent", "median")))
naive["yoy_pct"] = (naive.median_rent.pct_change() * PERCENT).round(2)
naive

TWO_BEDROOMS = 2

window = leases[leases.year.between(FIRST_YEAR_SHOWN, LAST_OBSERVED_YEAR)]
mix = window.groupby("year").agg(
    leases=("sRent", "size"),
    median_sqft=("sSqft", "median"),
    pct_2bed_plus=("sBeds", lambda beds: (beds >= TWO_BEDROOMS).mean() * PERCENT),
    buildings=("sBuilding", "nunique"))
mix["rent_per_sqft"] = (window.assign(psf=window.sRent / window.sSqft)
                        .groupby("year").psf.median())
mix.round(2)

SNAPSHOT_YEAR = 2024   # first year in which all three new buildings lease

# When did each building start generating leases?
print(window.groupby("sBuilding").year.min().sort_values().to_string())
print()
# And at what rent level? (leases of SNAPSHOT_YEAR)
print(window[window.year == SNAPSHOT_YEAR].groupby("sBuilding")
      .agg(leases=("sRent", "size"), median_rent=("sRent", "median"))
      .sort_values("median_rent").to_string())

sBuilding
Daniel-Johnson    2019
Levesque          2019
Saint-Elzear      2019
Le Carlyle        2023
The Met           2023
Westpark          2024

                leases  median_rent
sBuilding                          
Levesque            84       1672.5
Daniel-Johnson     136       1870.0
Saint-Elzear       215       2145.0
Westpark           144       2202.5
Le Carlyle         195       2350.0
The Met            128       2752.5

building_mix = pd.crosstab(validated_leases.sLeaseFrom.dt.year, validated_leases.sBuilding, normalize="index")
ax = building_mix.plot.area(stacked=True, figsize=(9, 4))
ax.set(title="Lease composition by building", ylabel="Share of the year's leases", xlabel="Lease start year")
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
plt.tight_layout()
plt.show()

COLLISION_EXAMPLE_SITE, COLLISION_EXAMPLE_UNIT = "saint-elzear", "0203"   # example given by the starter

site_key_collisions = listings[listings.duplicated(["sSite", "sUnitCode"], keep=False)]
print(f"{len(site_key_collisions):,} units out of {len(listings):,} collide on sSite + sUnitCode")
example = listings[(listings.sSite == COLLISION_EXAMPLE_SITE) & (listings.sUnitCode == COLLISION_EXAMPLE_UNIT)]
print(f"Example {COLLISION_EXAMPLE_SITE} / {COLLISION_EXAMPLE_UNIT}: {example.hUnit.nunique()} distinct apartments (hUnit), "
      f"{example.sPropCode.nunique()} different property codes")
print(f"Duplicates on sPropCode + sUnitCode in the listings: {listings.duplicated(UNIT_KEY).sum()}")

248 units out of 1,061 collide on sSite + sUnitCode
Example saint-elzear / 0203: 3 distinct apartments (hUnit), 3 different property codes
Duplicates on sPropCode + sUnitCode in the listings: 0

GAP_MIN_YEARS, GAP_MAX_YEARS = 0.5, 2.5   # shorter: often the same lease recorded twice; longer: too old
FIRST_PAIR_YEAR_SHOWN = 2021


def same_unit_growth(df, price_col="sRent", date_col="sLeaseFrom", min_gap=GAP_MIN_YEARS, max_gap=GAP_MAX_YEARS):
    # Starter version: annualised rent change, each unit compared with its own previous lease.
    d = df.sort_values(UNIT_KEY + [date_col]).copy()
    d["prev_price"] = d.groupby(UNIT_KEY)[price_col].shift(1)
    d["prev_date"] = d.groupby(UNIT_KEY)[date_col].shift(1)
    d = d.dropna(subset=["prev_price", "prev_date"])
    d["gap_years"] = (d[date_col] - d.prev_date).dt.days / DAYS_PER_YEAR
    d = d[d.gap_years.between(min_gap, max_gap, inclusive="neither")]
    d = d[(d.prev_price > 0) & (d[price_col] > 0)]
    d["growth_pct"] = ((d[price_col] / d.prev_price) ** (1 / d.gap_years) - 1) * PERCENT
    return d


pairs = same_unit_growth(leases)
print(f"{len(pairs):,} usable pairs across {pairs.groupby(UNIT_KEY).ngroups:,} units")

honest = (pairs[pairs.year.between(FIRST_PAIR_YEAR_SHOWN, LAST_OBSERVED_YEAR)]
          .groupby("year")
          .growth_pct.agg(pairs="size", median="median").round(2))
honest

3,179 usable pairs across 890 units

fig, ax = plt.subplots(figsize=(9, 4.5))
years_shown = range(FIRST_PAIR_YEAR_SHOWN, TARGET_YEAR)
ax.plot(years_shown, naive.loc[years_shown, "yoy_pct"], "o--", label="naive: median of all leases", lw=2)
ax.plot(years_shown, honest.loc[years_shown, "median"], "o-", label="same unit compared with itself", lw=2)
ax.axhline(0, color="grey", lw=0.8)
ax.set_ylabel("annual rent growth (%)")
ax.set_title("The naive method mostly measures composition")
ax.set_xticks(list(years_shown))
ax.legend()
ax.grid(alpha=0.3)
plt.tight_layout()
plt.show()

@dataclass(frozen=True)
class ForecastSpec:
    # Parameters frozen before the backtests of this revision.
    origin_month: int = 12                         # origin of each forecast: 31 December of the previous year
    origin_day: int = 31
    gap_min_years: float = GAP_MIN_YEARS           # admissible interval between two lease starts
    gap_max_years: float = GAP_MAX_YEARS
    growth_clip: tuple = (-0.30, 0.60)             # robustness bounds, training means only
    gap_clip: tuple = (0.0, 0.75)                  # bounds of the forecast concession gap
    shrink_k_cell: float = 10.0                    # shrinkage of building × event toward province × event
    shrink_k_mix: float = 20.0                     # shrinkage of the renewal share toward the province
    trailing_years_mix: int = 3                    # window of the renewal share
    components: tuple = ("DIRECT", "BRIDGE", "ANCHOR")
    component_weights: tuple = (1 / 3, 1 / 3, 1 / 3)
    face_persistence_weight: float = 0.5           # contractual: half last year, half history
    evaluation_years: tuple = (2020, 2021, 2022, 2023, 2024, 2025)


SPEC = ForecastSpec()
PAIR_COLUMNS = ["property_code", "unit_code", "province", "bedrooms", "term_seq", "prior_seq", "sign_date", "lease_start",
                "lease_end", "prior_start", "prior_sign", "event_year", "event_type", "prior_face_rent", "prior_effective_rent",
                "face_rent", "effective_rent", "interval_years", "face_growth", "effective_growth", "prior_gap", "new_gap", "eligible"]


def build_pairs(lease_table):
    # Pairs of consecutive leases of the same unit, annualised growth and concession gaps.
    ordered = validate_leases(lease_table).sort_values(UNIT_KEY + ["sTermSeq", "sLeaseFrom"])
    by_unit = ordered.groupby(UNIT_KEY, sort=False)
    paired = ordered.assign(prior_start=by_unit["sLeaseFrom"].shift(1), prior_sign=by_unit["sSignDate"].shift(1),
                            prior_face_rent=by_unit["sRent"].shift(1), prior_effective_rent=by_unit["sRentEffective"].shift(1),
                            prior_seq=by_unit["sTermSeq"].shift(1))
    paired = paired[paired["prior_start"].notna()].copy()
    paired["interval_years"] = (paired["sLeaseFrom"] - paired["prior_start"]).dt.days / DAYS_PER_YEAR
    paired["face_growth"] = (paired["sRent"] / paired["prior_face_rent"]) ** (1.0 / paired["interval_years"]) - 1.0
    paired["effective_growth"] = (paired["sRentEffective"] / paired["prior_effective_rent"]) ** (1.0 / paired["interval_years"]) - 1.0
    paired["prior_gap"] = 1.0 - paired["prior_effective_rent"] / paired["prior_face_rent"]
    paired["new_gap"] = 1.0 - paired["sRentEffective"] / paired["sRent"]
    rents_positive = (paired[["sRent", "prior_face_rent", "sRentEffective", "prior_effective_rent"]] > 0).all(axis=1)
    consecutive = paired["sTermSeq"].sub(paired["prior_seq"]).eq(1)
    paired["eligible"] = (paired["interval_years"].gt(SPEC.gap_min_years) & paired["interval_years"].lt(SPEC.gap_max_years)
                          & rents_positive & consecutive)
    paired["event_year"] = paired["sLeaseFrom"].dt.year
    paired["event_type"] = np.where(paired["sRenewal"].eq(1), "renewal", "turnover")
    return paired.rename(columns=COLUMN_RENAMES)[PAIR_COLUMNS].reset_index(drop=True)


def exact_bridge(face_growth, prior_gap, new_gap, interval_years=1.0):
    # Exact identity: (1 + g_contractual) × ((1 − new_gap) / (1 − previous_gap)) ** (1/t) − 1.
    inputs = (interval_years, prior_gap, new_gap, face_growth)
    if (any(not np.isfinite(np.asarray(value)).all() for value in inputs)
            or np.any(np.asarray(interval_years) <= 0) or np.any(np.asarray(prior_gap) >= 1)
            or np.any(np.asarray(prior_gap) < 0) or np.any(np.asarray(new_gap) < 0)
            or np.any(np.asarray(new_gap) >= 1) or np.any(np.asarray(face_growth) <= -1)):
        raise DataContractError("the bridge requires positive durations and valid rent ratios")
    return (1.0 + face_growth) * ((1.0 - new_gap) / (1.0 - prior_gap)) ** (1.0 / interval_years) - 1.0


unit_pairs = build_pairs(leases)
eligible_pairs = unit_pairs[unit_pairs["eligible"]]
exclusion_reasons = pd.Series({
    "interval ≤ 0.5 year": int((unit_pairs.interval_years <= SPEC.gap_min_years).sum()),
    "interval ≥ 2.5 years": int((unit_pairs.interval_years >= SPEC.gap_max_years).sum()),
    "non-consecutive sequences": int(unit_pairs.term_seq.sub(unit_pairs.prior_seq).ne(1).sum()),
}, name="pairs (one case can have two reasons)")
display(pd.DataFrame([{"candidate_pairs": len(unit_pairs), "eligible": len(eligible_pairs),
                       "excluded": len(unit_pairs) - len(eligible_pairs)}]))
display(exclusion_reasons)

assert eligible_pairs["term_seq"].sub(eligible_pairs["prior_seq"]).eq(1).all()
np.testing.assert_allclose(exact_bridge(eligible_pairs.face_growth, eligible_pairs.prior_gap,
                                        eligible_pairs.new_gap, eligible_pairs.interval_years),
                           eligible_pairs.effective_growth, atol=1e-12)
print("Contractual (face) / effective identity verified on all eligible pairs.")

interval ≤ 0.5 year          62
interval ≥ 2.5 years          0
non-consecutive sequences     0
Name: pairs (one case can have two reasons), dtype: int64

Contractual (face) / effective identity verified on all eligible pairs.

annual_growth = eligible_pairs.groupby("event_year").agg(
    pairs=("effective_growth", "size"),
    face_pct=("face_growth", lambda growth: PERCENT * growth.mean()),
    effective_pct=("effective_growth", lambda growth: PERCENT * growth.mean()),
    median_effective_pct=("effective_growth", lambda growth: PERCENT * growth.median()),
    renewal_share=("event_type", lambda event: event.eq("renewal").mean())).reset_index()
naive_median_change = validated_leases.groupby(validated_leases.sLeaseFrom.dt.year).sRent.median().pct_change() * PERCENT
annual_growth["naive_median_face_change_pct"] = annual_growth.event_year.map(naive_median_change)
display(annual_growth.round(3))

ax = annual_growth.set_index("event_year")[["naive_median_face_change_pct", "face_pct", "effective_pct"]].rename(columns={
    "naive_median_face_change_pct": "Naive median (all leases)", "face_pct": "Same unit, contractual",
    "effective_pct": "Same unit, effective"}).plot(marker="o", figsize=(9, 4))
ax.set(title="Naive median vs same-unit growth", ylabel="% annualised / annual change", xlabel="Start year of the new lease")
ax.axhline(0, color="grey", lw=0.8)
plt.tight_layout()
plt.show()

conc = (leases[leases.year.between(FIRST_PAIR_YEAR_SHOWN, LAST_OBSERVED_YEAR)]
        .groupby("year")
        .agg(leases=("sRent", "size"),
             pct_with_concession=("sConcession", lambda flag: flag.mean() * PERCENT),
             median_contracted=("sRent", "median"),
             median_effective=("sRentEffective", "median")))
conc["gap_pct"] = ((conc.median_effective / conc.median_contracted - 1) * PERCENT)
conc.round(2)

# Does the concession change the trend, or only the level?
eff = same_unit_growth(leases, price_col="sRentEffective")
compare = pd.DataFrame({
    "contracted": honest["median"],
    "effective": eff[eff.year.between(FIRST_PAIR_YEAR_SHOWN, LAST_OBSERVED_YEAR)].groupby("year").growth_pct.median(),
}).round(2)
compare

RENT_CREDIT_CODES = ("promopay", "freerent")
AMENITY_CODES = ("freepark", "freelock", "freeappl")


def match_concessions_to_leases(lease_table, concession_table):
    # Attach each entry to the lease of the same unit that contains its start date.
    lease_rows = validate_leases(lease_table).reset_index(drop=True).reset_index(names="lease_id")
    ledger = normalise_unit_key(concession_table).reset_index(names="concession_id")
    for column in ("sDateFrom", "sDateTo"):
        ledger[column] = pd.to_datetime(ledger[column])
    matched = ledger.merge(lease_rows[["lease_id", *UNIT_KEY, "sLeaseFrom", "sLeaseTo"]], on=UNIT_KEY, how="left")
    matched = matched[matched.sDateFrom.between(matched.sLeaseFrom, matched.sLeaseTo)].copy()
    if matched.groupby("concession_id").lease_id.nunique().gt(1).any():
        raise DataContractError("a concession entry matches several leases")
    charge_code = matched.sChargeCode.str.lower()
    matched["code"] = charge_code
    matched["category"] = np.select([charge_code.isin(RENT_CREDIT_CODES), charge_code.isin(AMENITY_CODES)],
                                    ["rent", "amenity"], default="other")
    matched["total_credit"] = -matched.sAmount * matched.sMonths
    if (matched.total_credit < 0).any():
        raise DataContractError("a concession entry is a positive charge")
    return lease_rows, matched


lease_rows, matched_concessions = match_concessions_to_leases(leases, concessions)
rent_credit_by_lease = matched_concessions[matched_concessions.category.eq("rent")].groupby("lease_id").total_credit.sum()
entries_per_lease = matched_concessions.groupby("lease_id").size()
lease_rows["matched_entries"] = lease_rows.lease_id.map(entries_per_lease).fillna(0)
lease_rows["rent_only"] = lease_rows.sRent - lease_rows.lease_id.map(rent_credit_by_lease).fillna(0) / lease_rows.sTermMonths
lease_rows["flag_consistent"] = ~(lease_rows.sConcession.eq(1) & lease_rows.matched_entries.eq(0))
by_unit_sequence = lease_rows.sort_values(UNIT_KEY + ["sTermSeq"]).groupby(UNIT_KEY)
lease_rows["prior_rent_only"] = by_unit_sequence.rent_only.shift()
lease_rows["prior_flag_consistent"] = by_unit_sequence.flag_consistent.shift(fill_value=False)

concession_reconciliation = pd.DataFrame([{
    "entries": len(concessions),
    "matched to a lease": matched_concessions.concession_id.nunique(),
    "unmatched": len(concessions) - matched_concessions.concession_id.nunique(),
    "leases flagged without an entry": int((lease_rows.sConcession.eq(1) & lease_rows.matched_entries.eq(0)).sum()),
    "leases not flagged with an entry": int((lease_rows.sConcession.eq(0) & lease_rows.matched_entries.gt(0)).sum())}])
display(concession_reconciliation)

pairs_with_reconstruction = eligible_pairs.merge(
    lease_rows[[*UNIT_KEY, "sTermSeq", "rent_only", "prior_rent_only", "flag_consistent", "prior_flag_consistent"]],
    left_on=["property_code", "unit_code", "term_seq"], right_on=[*UNIT_KEY, "sTermSeq"], validate="one_to_one")
common_sample = pairs_with_reconstruction[pairs_with_reconstruction.flag_consistent & pairs_with_reconstruction.prior_flag_consistent
                                          & pairs_with_reconstruction.rent_only.gt(0)
                                          & pairs_with_reconstruction.prior_rent_only.gt(0)].copy()
common_sample["rent_only_pct"] = PERCENT * ((common_sample.rent_only / common_sample.prior_rent_only) ** (1 / common_sample.interval_years) - 1)
common_sample_by_year = common_sample.groupby("event_year").agg(
    n=("effective_growth", "size"), face_pct=("face_growth", lambda growth: PERCENT * growth.mean()),
    rent_only_pct=("rent_only_pct", "mean"),
    supplied_effective_pct=("effective_growth", lambda growth: PERCENT * growth.mean())).reset_index()
display(common_sample_by_year.round(3))
ax = common_sample_by_year.set_index("event_year")[["face_pct", "rent_only_pct", "supplied_effective_pct"]].rename(columns={
    "face_pct": "Contractual", "rent_only_pct": "Contractual − ledger rent credits",
    "supplied_effective_pct": "Supplied effective"}).plot(marker="o", figsize=(9, 4))
ax.set(title="Same sample: effect of the rent definition", ylabel="Annualised growth (%)", xlabel="Start year of the new lease")
plt.tight_layout()
plt.show()

def concession_divergence(pairs_table):
    # By year: contractual vs effective growth, and change in the gap = incidence + depth.
    rows = []
    for year, year_pairs in pairs_table[pairs_table["eligible"]].groupby("event_year"):
        incidence_prior, incidence_new = float((year_pairs["prior_gap"] > 0).mean()), float((year_pairs["new_gap"] > 0).mean())
        depth_prior = float(year_pairs.loc[year_pairs["prior_gap"] > 0, "prior_gap"].mean()) if incidence_prior else 0.0
        depth_new = float(year_pairs.loc[year_pairs["new_gap"] > 0, "new_gap"].mean()) if incidence_new else 0.0
        gap_prior, gap_new = float(year_pairs["prior_gap"].mean()), float(year_pairs["new_gap"].mean())
        incidence_part = (incidence_new - incidence_prior) * (depth_prior + depth_new) / 2.0
        depth_part = (depth_new - depth_prior) * (incidence_prior + incidence_new) / 2.0
        rows.append({
            "event_year": int(year), "pairs": len(year_pairs),
            "face_pct": PERCENT * year_pairs["face_growth"].mean(), "effective_pct": PERCENT * year_pairs["effective_growth"].mean(),
            "face_minus_effective_pp": PERCENT * (year_pairs["face_growth"].mean() - year_pairs["effective_growth"].mean()),
            "prior_gap_pct": PERCENT * gap_prior, "new_gap_pct": PERCENT * gap_new, "gap_change_pp": PERCENT * (gap_new - gap_prior),
            "incidence_prior": incidence_prior, "incidence_new": incidence_new,
            "depth_prior_pct": PERCENT * depth_prior, "depth_new_pct": PERCENT * depth_new,
            "gap_change_from_incidence_pp": PERCENT * incidence_part, "gap_change_from_depth_pp": PERCENT * depth_part})
    return pd.DataFrame(rows)


def concession_code_contribution(lease_rows_table, matched_table):
    # Average amortised concession as % of contractual rent, over all leases, by year and by code.
    credit_by_code = matched_table.groupby(["lease_id", "code"])["total_credit"].sum().unstack(fill_value=0.0)
    codes = list(credit_by_code.columns)
    per_lease = lease_rows_table[["lease_id", "sLeaseFrom", "sTermMonths", "sRent", "sRentEffective"]].join(credit_by_code, on="lease_id")
    per_lease[codes] = per_lease[codes].fillna(0.0).div(per_lease["sTermMonths"], axis=0).div(per_lease["sRent"], axis=0)
    start_year = per_lease["sLeaseFrom"].dt.year
    contribution = per_lease.groupby(start_year)[codes].mean().mul(PERCENT)
    contribution["total_matched_pct"] = contribution[codes].sum(axis=1)
    contribution["supplied_gap_pct"] = PERCENT * (1 - per_lease["sRentEffective"] / per_lease["sRent"]).groupby(start_year).mean()
    contribution["unexplained_by_ledger_pct"] = contribution["supplied_gap_pct"] - contribution["total_matched_pct"]
    contribution.index.name = "lease_start_year"
    return contribution.reset_index()


divergence = concession_divergence(unit_pairs)
display(divergence.round(2))
fig, axes = plt.subplots(1, 2, figsize=(13, 4))
divergence.set_index("event_year")[["face_pct", "effective_pct"]].rename(
    columns={"face_pct": "Contractual", "effective_pct": "Effective"}).plot(ax=axes[0], marker="o", title="Contractual vs effective, same unit")
axes[0].set(ylabel="% annualised", xlabel="Start year of the new lease")
divergence.set_index("event_year")[["gap_change_from_incidence_pp", "gap_change_from_depth_pp"]].rename(
    columns={"gap_change_from_incidence_pp": "Incidence", "gap_change_from_depth_pp": "Depth"}).plot.bar(
    ax=axes[1], stacked=True, title="Change in the concession gap: incidence + depth")
axes[1].set(ylabel="percentage points", xlabel="Start year of the new lease")
plt.tight_layout()
plt.show()
display(concession_code_contribution(lease_rows, matched_concessions).round(2))

FIRST_SPLIT_YEAR = 2022

split = (pairs[pairs.year.between(FIRST_SPLIT_YEAR, LAST_OBSERVED_YEAR)]
         .groupby(["year", "sRenewal"])
         .growth_pct.agg(pairs="size", median="median").round(2)
         .unstack("sRenewal"))
split.columns = [f"{stat}_{'renewal' if is_renewal else 'turnover'}" for stat, is_renewal in split.columns]
split

segments = eligible_pairs.groupby(["event_year", "province", "event_type"]).agg(
    n=("effective_growth", "size"),
    face_pct=("face_growth", lambda growth: PERCENT * growth.mean()),
    effective_pct=("effective_growth", lambda growth: PERCENT * growth.mean())).reset_index()
display(segments[segments.event_year.ge(REQUIRED_BACKTEST_YEARS[0])].round(3))

asking_known = asking.copy()
asking_known["sMonth"] = pd.to_datetime(asking_known.sMonth)
asking_known = asking_known[asking_known.sMonth.le(pd.Timestamp(LAST_OBSERVED_YEAR, 12, 31))]
latest_asking = asking_known.sort_values("sMonth").drop_duplicates(UNIT_KEY, keep="last")
asking_vs_listing = latest_asking.merge(listings[UNIT_KEY + ["sRent"]], on=UNIT_KEY, validate="one_to_one")
display(pd.DataFrame([{"origin": f"{LAST_OBSERVED_YEAR}-12-31", "units with asking rent": len(latest_asking),
                       "matched to listings": len(asking_vs_listing),
                       "median asking / listing ratio": float((asking_vs_listing.sAskingRent / asking_vs_listing.sRent).median())}]).round(4))

external = pd.read_csv("external_context.csv")
external_post_origin = pd.read_csv("external_postorigin_2026.csv")
EXTERNAL_REQUIRED_COLUMNS = {"target_year", "release_date", "source_url", "metric", "value"}


def origin_date(target_year):
    # Forecast origin: 31 December of the year before the target.
    return pd.Timestamp(target_year - 1, SPEC.origin_month, SPEC.origin_day)


def external_asof(external_table, target_year):
    # Public sources available at the origin; rejects later publications or those without provenance.
    if external_table is None:
        return pd.DataFrame()
    if not isinstance(external_table, pd.DataFrame):
        raise DataContractError("external must be a DataFrame of public sources with provenance")
    if EXTERNAL_REQUIRED_COLUMNS - set(external_table):
        raise DataContractError(f"external: missing columns {sorted(EXTERNAL_REQUIRED_COLUMNS - set(external_table))}")
    available = external_table[external_table.target_year.eq(target_year)].copy()
    release = pd.to_datetime(available.release_date, errors="coerce")
    if release.isna().any() or (release > origin_date(target_year)).any():
        raise DataContractError("publication missing or later than the forecast origin")
    if available.source_url.isna().any() or not available.source_url.astype(str).str.match(r"https?://\S+").all():
        raise DataContractError("missing provenance")
    if not np.isfinite(pd.to_numeric(available.value, errors="coerce")).all():
        raise DataContractError("non-finite public values")
    return available


display(external_asof(external, TARGET_YEAR)[["geography", "metric", "reference_period", "release_date", "value", "unit"]])

@dataclass(frozen=True)
class OriginBundle:
    # Everything the forecast is allowed to see for a target year.
    target_year: int
    origin: pd.Timestamp
    pairs: pd.DataFrame     # eligible pairs signed and started before the origin
    roster: pd.DataFrame    # units whose last known lease ends during the target year

    @property
    def summary(self):
        return {"target_year": self.target_year, "origin": self.origin.date().isoformat(),
                "n_training_pairs": int(len(self.pairs)), "n_roster_units": int(len(self.roster)),
                "training_years": sorted(int(year) for year in self.pairs["event_year"].unique())}


ROSTER_COLUMNS = ["property_code", "unit_code", "province", "bedrooms", "term_seq", "lease_end", "face_rent", "effective_rent",
                  "prior_gap", "expected_years", "comparable"]


def build_roster(lease_table, target_year):
    # Units whose last known lease at the origin ends during the target year.
    # The origin filter comes BEFORE the choice of the last lease: a more recent extract does not change the roster.
    checked = validate_leases(lease_table)
    origin = origin_date(target_year)
    known_at_origin = checked[(checked["sSignDate"] <= origin) & (checked["sLeaseFrom"] <= origin)]
    latest_lease = known_at_origin.sort_values(UNIT_KEY + ["sTermSeq", "sLeaseFrom"]).drop_duplicates(UNIT_KEY, keep="last")
    roster = latest_lease[latest_lease["sLeaseTo"].dt.year.eq(target_year)].rename(columns=COLUMN_RENAMES)
    roster["prior_gap"] = 1.0 - roster["effective_rent"] / roster["face_rent"]
    # Assumption: the next lease starts the day after the current lease ends.
    roster["expected_years"] = ((roster["lease_end"] + pd.Timedelta(days=1)) - roster["lease_start"]).dt.days / DAYS_PER_YEAR
    roster["comparable"] = roster["expected_years"].between(SPEC.gap_min_years, SPEC.gap_max_years, inclusive="neither")
    return roster[ROSTER_COLUMNS].reset_index(drop=True)


def make_bundle(lease_table, target_year):
    # Filter on signature AND start before pairing: a missing sequence cannot be bridged.
    checked = validate_leases(lease_table)
    origin = origin_date(target_year)
    observed = checked[(checked["sSignDate"] <= origin) & (checked["sLeaseFrom"] <= origin)]
    observed_pairs = build_pairs(observed)
    known_pairs = observed_pairs[observed_pairs["eligible"]].copy()
    roster = build_roster(checked, target_year)
    if known_pairs.empty:
        raise DataContractError(f"no eligible pair known before {origin.date()}")
    if roster.empty or not roster["comparable"].any():
        raise DataContractError(f"no unit expiring in {target_year} is known at {origin.date()}")
    return OriginBundle(target_year, origin, known_pairs, roster)


def clip_for_training(values, column):
    # Robustness bounds: forecast concession gap, or growth.
    lower, upper = SPEC.gap_clip if column == "new_gap" else SPEC.growth_clip
    return values.clip(lower, upper)


def shrunk_cell_means(pairs_table, column, shrink_k):
    # Building × event means shrunk toward province × event (then event, then portfolio).
    if pairs_table.empty:
        return {}
    bounded = pairs_table.assign(_value=clip_for_training(pairs_table[column], column))
    portfolio_mean = float(bounded["_value"].mean())
    event_means = bounded.groupby("event_type")["_value"].mean().to_dict()
    province_event_means = bounded.groupby(["province", "event_type"])["_value"].mean()
    province_of = bounded.groupby("property_code")["province"].first().to_dict()
    cell_means = {}
    for (property_code, event_type), cell in bounded.groupby(["property_code", "event_type"]):
        parent = province_event_means.get((province_of[property_code], event_type), event_means.get(event_type, portfolio_mean))
        n_pairs = len(cell)
        cell_means[(property_code, event_type)] = (n_pairs * float(cell["_value"].mean()) + shrink_k * parent) / (n_pairs + shrink_k)
    return cell_means


def cell_value(cell_means, pairs_table, column, property_code, event_type, province):
    # Cell value, with hierarchical fallback for a missing cell.
    if (property_code, event_type) in cell_means:
        return cell_means[(property_code, event_type)]
    bounded = pairs_table.assign(_value=clip_for_training(pairs_table[column], column))
    province_event = bounded[bounded["province"].eq(province) & bounded["event_type"].eq(event_type)]["_value"]
    if len(province_event):
        return float(province_event.mean())
    same_event = bounded[bounded["event_type"].eq(event_type)]["_value"]
    return float(same_event.mean()) if len(same_event) else float(bounded["_value"].mean())


def event_mix(bundle):
    # Expected share of renewals / turnovers by building, shrunk toward the province.
    roster = bundle.roster[bundle.roster["comparable"]]
    recent = bundle.pairs[bundle.pairs["event_year"].between(bundle.target_year - SPEC.trailing_years_mix, bundle.target_year - 1)]
    overall_rate = (float(recent["event_type"].eq("renewal").mean()) if len(recent)
                    else float(bundle.pairs["event_type"].eq("renewal").mean()))
    rows = []
    for property_code, units in roster.groupby("property_code"):
        province = units["province"].iloc[0]
        property_recent = recent[recent["property_code"].eq(property_code)]
        province_recent = recent[recent["province"].eq(province)]
        prior_rate = float(province_recent["event_type"].eq("renewal").mean()) if len(province_recent) else overall_rate
        renewal_rate = ((property_recent["event_type"].eq("renewal").sum() + SPEC.shrink_k_mix * prior_rate)
                        / (len(property_recent) + SPEC.shrink_k_mix))
        for event_type, share in (("renewal", renewal_rate), ("turnover", 1.0 - renewal_rate)):
            rows.append({"property_code": property_code, "province": province, "event_type": event_type,
                         "roster_units": len(units), "event_share": float(share), "expected_units": len(units) * float(share),
                         "roster_prior_gap": float(units["prior_gap"].mean()), "n_recent_pairs": int(len(property_recent))})
    mix_table = pd.DataFrame(rows)
    mix_table["weight"] = mix_table["expected_units"] / mix_table["expected_units"].sum()
    return mix_table


def roster_bridge(bundle, property_code, face_growth, new_gap):
    # Average of the unit-by-unit identities over a building's roster (next start = current end + 1 day).
    units = bundle.roster[bundle.roster["property_code"].eq(property_code) & bundle.roster["comparable"]]
    return float(np.mean(exact_bridge(face_growth, units["prior_gap"], new_gap, units["expected_years"])))

def component_cells(bundle):
    # DIRECT, BRIDGE, ANCHOR and contractual for each building × event.
    known_pairs, target_year = bundle.pairs, bundle.target_year
    last_year = known_pairs[known_pairs["event_year"].eq(target_year - 1)]
    if last_year.empty:   # no lease started last year: most recent year available
        last_year = known_pairs[known_pairs["event_year"].eq(known_pairs["event_year"].max())]
    weights = event_mix(bundle)
    direct = shrunk_cell_means(last_year, "effective_growth", SPEC.shrink_k_cell)
    face_last = shrunk_cell_means(last_year, "face_growth", SPEC.shrink_k_cell)
    gap_last = shrunk_cell_means(last_year, "new_gap", SPEC.shrink_k_cell)
    anchor = shrunk_cell_means(known_pairs, "effective_growth", SPEC.shrink_k_cell)
    face_anchor = shrunk_cell_means(known_pairs, "face_growth", SPEC.shrink_k_cell)
    rows = []
    for _, cell in weights.iterrows():
        key = (cell["property_code"], cell["event_type"], cell["province"])
        face_persistence = cell_value(face_last, last_year, "face_growth", *key)
        new_gap_persistence = cell_value(gap_last, last_year, "new_gap", *key)
        rows.append({**cell.to_dict(),
                     "DIRECT": cell_value(direct, last_year, "effective_growth", *key),
                     "BRIDGE": roster_bridge(bundle, cell["property_code"], face_persistence, new_gap_persistence),
                     "ANCHOR": cell_value(anchor, known_pairs, "effective_growth", *key),
                     "face_persistence": face_persistence, "face_anchor": cell_value(face_anchor, known_pairs, "face_growth", *key),
                     "new_gap_persistence": new_gap_persistence,
                     "n_last_year_pairs": int((last_year["property_code"].eq(cell["property_code"])
                                               & last_year["event_type"].eq(cell["event_type"])).sum())})
    return pd.DataFrame(rows)


def combine(cells, components=None, weights=None):
    # Combination with fixed weights: effective = average of the components; contractual = last year / history blend.
    components = SPEC.components if components is None else components
    weights = SPEC.component_weights if weights is None else weights
    if (not components or len(components) != len(weights) or not np.isfinite(weights).all() or min(weights) < 0
            or abs(sum(weights) - 1.0) > 1e-9 or len(set(components)) != len(components)
            or not set(components).issubset(SPEC.components)):
        raise DataContractError("the weights must match the components and sum to 1")
    combined = cells.copy()
    combined["effective"] = sum(weight * combined[name] for name, weight in zip(components, weights))
    combined["face"] = (SPEC.face_persistence_weight * combined["face_persistence"]
                        + (1 - SPEC.face_persistence_weight) * combined["face_anchor"])
    return combined


def aggregate(cells, column="effective"):
    # Average weighted by each cell's expected weight.
    return float(np.average(cells[column], weights=cells["weight"]))


def forecast(bundle, components=None, weights=None):
    cells = combine(component_cells(bundle), components, weights)
    return {"target_year": bundle.target_year, "origin": bundle.origin.date().isoformat(),
            "effective": aggregate(cells, "effective"), "face": aggregate(cells, "face"),
            "components": {name: aggregate(cells, name) for name in SPEC.components},
            "renewal_share": float(cells.loc[cells["event_type"].eq("renewal"), "weight"].sum()),
            "roster_units": int(bundle.roster["comparable"].sum()),
            "roster_excluded": int((~bundle.roster["comparable"]).sum()),
            "cells": cells, "bundle": bundle.summary}

def realized(lease_table, target_year):
    # What actually happened: all eligible pairs whose new lease starts during the target year.
    all_pairs = build_pairs(lease_table)
    return all_pairs[all_pairs["eligible"] & all_pairs["event_year"].eq(target_year)].copy()


def backtest_detail(lease_table, target_year, components=None, weights=None):
    bundle = make_bundle(lease_table, target_year)
    prediction = forecast(bundle, components, weights)
    actual_pairs = realized(lease_table, target_year)
    if actual_pairs.empty:
        raise DataContractError(f"no realised pair in {target_year}")
    actual_effective, actual_face = float(actual_pairs["effective_growth"].mean()), float(actual_pairs["face_growth"].mean())
    actual_by_cell = actual_pairs.groupby(["property_code", "event_type"]).agg(
        actual_effective=("effective_growth", "mean"), actual_face=("face_growth", "mean"),
        realized_n=("effective_growth", "size")).reset_index()
    cells = prediction["cells"].merge(actual_by_cell, on=["property_code", "event_type"], how="left")
    cells["abs_error"] = (cells["effective"] - cells["actual_effective"]).abs()
    scored = cells.dropna(subset=["actual_effective"])
    segment_mae = float(np.average(scored["abs_error"], weights=scored["realized_n"])) if len(scored) else np.nan
    # Diagnostics only: the roster is never reweighted with the realised mix.
    comparable_roster = bundle.roster[bundle.roster.comparable]
    in_roster = actual_pairs.merge(comparable_roster[["property_code", "unit_code", "term_seq"]],
                                   left_on=["property_code", "unit_code", "prior_seq"],
                                   right_on=["property_code", "unit_code", "term_seq"], how="inner", validate="many_to_one")
    roster_units_seen = in_roster[["property_code", "unit_code"]].drop_duplicates().shape[0]
    return {"target_year": target_year, "origin": prediction["origin"], "predicted": prediction["effective"],
            "actual": actual_effective, "error": prediction["effective"] - actual_effective,
            "predicted_face": prediction["face"], "actual_face": actual_face, "face_error": prediction["face"] - actual_face,
            "components": prediction["components"],
            "component_errors": {name: value - actual_effective for name, value in prediction["components"].items()},
            "segment_mae": segment_mae, "realized_pairs": int(len(actual_pairs)), "roster_units": prediction["roster_units"],
            "realized_in_roster": int(len(in_roster)), "target_roster_coverage": len(in_roster) / len(actual_pairs),
            "roster_observed_fraction": roster_units_seen / len(comparable_roster),
            "segment_target_coverage": float(scored.realized_n.sum() / len(actual_pairs)),
            "by_segment": cells.to_dict("records"), "cells": cells, "bundle": prediction["bundle"]}


def estimate_2026(leases, asking, external=None):
    # Returns our estimate of the 2026 rent increase, as a percentage.
    # Definition: average of the annualised effective-rent (sRentEffective) growth rates, same unit (sPropCode + sUnitCode),
    # consecutive leases whose new lease starts in 2026. `asking` is accepted to respect the starter's signature:
    # the asking rent remains context (section 6). `external` is checked by the date rule of section 7.2.
    external_asof(external, TARGET_YEAR)
    return PERCENT * forecast(make_bundle(leases, TARGET_YEAR))["effective"]


def backtest(leases, target_year):
    # Runs the method as if we were at the end of target_year - 1, and compares with what happened in target_year.
    detail = backtest_detail(leases, target_year)
    return {"target_year": target_year, "predicted_pct": PERCENT * detail["predicted"],
            "actual_pct": PERCENT * detail["actual"], "error_pp": PERCENT * detail["error"],
            "by_segment": detail["by_segment"], "segment_rate_units": "fraction"}


estimate_pct = estimate_2026(leases, asking, external)
forecast_2026 = forecast(make_bundle(leases, TARGET_YEAR))
display(pd.Series({"2026 effective increase (%)": estimate_pct,
                   "2026 contractual (face) increase (%)": PERCENT * forecast_2026["face"],
                   **{f"Component {name} (%)": PERCENT * value for name, value in forecast_2026["components"].items()},
                   "Expected renewal share": forecast_2026["renewal_share"],
                   "Comparable roster units": forecast_2026["roster_units"],
                   "Excluded units (term outside 0.5–2.5 years)": forecast_2026["roster_excluded"]}).round(4))

2026 effective increase (%)                      4.8047
2026 contractual (face) increase (%)             6.3780
Component DIRECT (%)                             3.8592
Component BRIDGE (%)                             7.5240
Component ANCHOR (%)                             3.0309
Expected renewal share                           0.6031
Comparable roster units                        928.0000
Excluded units (term outside 0.5–2.5 years)      3.0000
dtype: float64

def rolling_evaluation(lease_table, target_years, components=None, weights=None):
    rows = []
    for year in target_years:
        detail = backtest_detail(lease_table, year, components, weights)
        rows.append({"target_year": year, "origin": detail["origin"],
                     "predicted_pct": PERCENT * detail["predicted"], "actual_pct": PERCENT * detail["actual"],
                     "error_pp": PERCENT * detail["error"], "abs_error_pp": PERCENT * abs(detail["error"]),
                     "predicted_face_pct": PERCENT * detail["predicted_face"], "actual_face_pct": PERCENT * detail["actual_face"],
                     "face_error_pp": PERCENT * detail["face_error"], "segment_mae_pp": PERCENT * detail["segment_mae"],
                     **{f"{name}_pct": PERCENT * value for name, value in detail["components"].items()},
                     **{f"{name}_error_pp": PERCENT * value for name, value in detail["component_errors"].items()},
                     "realized_pairs": detail["realized_pairs"], "roster_units": detail["roster_units"],
                     **{key: detail[key] for key in ["realized_in_roster", "target_roster_coverage",
                                                     "roster_observed_fraction", "segment_target_coverage"]}})
    return pd.DataFrame(rows)


def baseline_evaluation(lease_table, target_years):
    # Three simple comparators, same origin and same target; no winner is chosen.
    rows = []
    for year in target_years:
        bundle = make_bundle(lease_table, year)
        actual = float(realized(lease_table, year)["effective_growth"].mean())
        for name, training in (("Last year pooled", bundle.pairs[bundle.pairs.event_year.eq(year - 1)]),
                               ("Trailing 3 years pooled", bundle.pairs[bundle.pairs.event_year.ge(year - 3)]),
                               ("All history pooled", bundle.pairs)):
            if training.empty:
                training = bundle.pairs
            prediction = float(training.effective_growth.mean())
            rows.append({"target_year": year, "model": name, "predicted_pct": PERCENT * prediction,
                         "actual_pct": PERCENT * actual, "error_pp": PERCENT * (prediction - actual)})
    return pd.DataFrame(rows)


required_backtests = pd.DataFrame([{key: value for key, value in backtest(leases, year).items()
                                    if key not in ("by_segment", "segment_rate_units")} for year in REQUIRED_BACKTEST_YEARS])
display(required_backtests.round(3))

evaluation_years = tuple(year for year in SPEC.evaluation_years if year < TARGET_YEAR)
all_folds = rolling_evaluation(leases, evaluation_years)
display(all_folds[["target_year", "predicted_pct", "actual_pct", "error_pp", "predicted_face_pct", "actual_face_pct",
                   "face_error_pp", "segment_mae_pp", "roster_units", "realized_pairs"]].round(3))

baselines = baseline_evaluation(leases, evaluation_years)
scores = baselines.groupby("model").error_pp.agg(MAE_pp=lambda error: error.abs().mean(), bias_pp="mean",
                                                 worst_pp=lambda error: error.abs().max())
scores.loc["Our method (fixed)"] = [all_folds.abs_error_pp.mean(), all_folds.error_pp.mean(), all_folds.abs_error_pp.max()]
scores["MAE_2023_2025_pp"] = baselines[baselines.target_year.isin(REQUIRED_BACKTEST_YEARS)].groupby("model").error_pp.apply(
    lambda error: error.abs().mean())
scores.loc["Our method (fixed)", "MAE_2023_2025_pp"] = all_folds[all_folds.target_year.isin(REQUIRED_BACKTEST_YEARS)].abs_error_pp.mean()
display(scores.round(3))

ax = all_folds.set_index("target_year")[["predicted_pct", "actual_pct"]].rename(
    columns={"predicted_pct": "Forecast at the previous 31 December", "actual_pct": "Realised"}).plot(marker="o", figsize=(9, 4))
ax.set(title="Backtest: simulated forecast vs realised (effective, same unit)", ylabel="% annualised", xlabel="Target year",
       xticks=list(evaluation_years))
plt.tight_layout()
plt.show()

display(all_folds[["target_year", "realized_in_roster", "target_roster_coverage", "roster_observed_fraction",
                   "segment_target_coverage"]].round(3))

def external_comparison(lease_table, external_table, target_year=TARGET_YEAR):
    cells = forecast(make_bundle(lease_table, target_year))["cells"]
    rows = []
    for source in external_asof(external_table, target_year).itertuples():
        geography = source.geography.lower()
        province = "Ontario" if ("ontario" in geography or "ottawa" in geography) else "Quebec"   # regimes never mixed
        province_cells = cells[cells.province.eq(province)]
        regulatory = "TAL" in source.metric or "guideline" in source.metric.lower()
        if regulatory:
            province_cells = province_cells[province_cells.event_type.eq("renewal")]
        face = PERCENT * np.average(province_cells.face, weights=province_cells.weight)
        effective = PERCENT * np.average(province_cells.effective, weights=province_cells.weight)
        is_vacancy = "vacancy" in source.metric.lower()
        rows.append({"geography": source.geography, "metric": source.metric, "reference_period": source.reference_period,
                     "release_date": source.release_date, "public_value": source.value,
                     "internal_scope": "renewal" if regulatory else "all events",
                     "internal_face_pct": face, "internal_effective_pct": effective,
                     "face_minus_benchmark_pp": np.nan if is_vacancy else face - source.value})
    return pd.DataFrame(rows)


display(external_comparison(leases, external).round(3))

FIRST_MARKET_YEAR = 2021
REFERENCE_YEAR_PATTERN = r"(\d{4})"
MARKET_PANELS = (("Quebec", "Montreal CMA", "Montreal CMA", "TAL", "Quebec (5 buildings)"),
                 ("Ontario", "Ottawa-Gatineau CMA (Ontario part)", "Ontario", "guideline", "Ontario (The Met)"))

internal_by_province = eligible_pairs.groupby(["province", "event_year"]).agg(
    face_pct=("face_growth", lambda growth: PERCENT * growth.mean()),
    effective_pct=("effective_growth", lambda growth: PERCENT * growth.mean()))
forecast_cells = forecast_2026["cells"]
public_series = external.assign(reference_year=external.reference_period.str.extract(REFERENCE_YEAR_PATTERN, expand=False).astype(int))

fig, axes = plt.subplots(1, 2, figsize=(14, 4.5), sharey=True)
for ax, (province, cma, cpi_geography, regulator, title) in zip(axes, MARKET_PANELS):
    internal = internal_by_province.loc[province].loc[FIRST_MARKET_YEAR:LAST_OBSERVED_YEAR]
    province_cells = forecast_cells[forecast_cells.province.eq(province)]
    ax.plot(internal.index, internal.face_pct, "o-", color="tab:blue", label="Équinoxe, contractual (same unit)")
    ax.plot(internal.index, internal.effective_pct, "o-", color="tab:orange", label="Équinoxe, effective (same unit)")
    ax.plot([TARGET_YEAR], [PERCENT * np.average(province_cells.face, weights=province_cells.weight)], "D", color="tab:blue", ms=9)
    ax.plot([TARGET_YEAR], [PERCENT * np.average(province_cells.effective, weights=province_cells.weight)], "D", color="tab:orange",
            ms=9, label="Our 2026 forecast")
    cmhc = public_series[public_series.geography.eq(cma) & public_series.metric.str.contains("fixed-sample")]
    cpi = public_series[public_series.geography.eq(cpi_geography) & public_series.metric.str.contains("CPI")]
    rules = public_series[public_series.metric.str.contains(regulator)]
    ax.plot(cmhc.reference_year, cmhc.value, "s--", color="tab:green", label="CMHC, fixed-sample average rent (CMA)")
    ax.plot(cpi.reference_year, cpi.value, "^--", color="tab:purple", label="CPI rent component (Statistics Canada)")
    ax.plot(rules.reference_year, rules.value, "x:", color="black", ms=8, label="Renewal regulation (TAL / guideline)")
    ax.set(title=title, xlabel="Year", xticks=list(range(FIRST_MARKET_YEAR, TARGET_YEAR + 1)))
    ax.grid(alpha=0.3)
axes[0].set_ylabel("% per year")
axes[1].legend(loc="upper left", bbox_to_anchor=(1.02, 1), fontsize=9)
plt.tight_layout()
plt.show()

def tal_passthrough_test(lease_table, external_table, test_years=REQUIRED_BACKTEST_YEARS):
    is_tal = external_table["metric"].str.contains("TAL", na=False)
    tal_by_year = {int(str(period).split()[-1]): float(value)
                   for period, value in zip(external_table.loc[is_tal, "reference_period"], external_table.loc[is_tal, "value"])}
    all_pairs = build_pairs(lease_table)
    quebec_renewals = all_pairs[all_pairs["eligible"] & all_pairs["province"].eq("Quebec") & all_pairs["event_type"].eq("renewal")]
    actual_by_year = PERCENT * quebec_renewals.groupby("event_year")["face_growth"].mean()
    rows = []
    for year in test_years:
        if year not in tal_by_year or year - 1 not in tal_by_year:
            raise DataContractError(f"TAL values for {year - 1} and {year} required")
        earlier = actual_by_year[actual_by_year.index < year]
        rows.append({"year": year, "actual_pct": float(actual_by_year[year]), "all_prior_mean_pct": float(earlier.mean()),
                     "persistence_pct": float(earlier.iloc[-1]),
                     "tal_passthrough_pct": float(earlier.iloc[-1]) + tal_by_year[year] - tal_by_year[year - 1],
                     "tal_pct": tal_by_year[year]})
    table = pd.DataFrame(rows)
    errors = {method: (table[f"{method}_pct"] - table["actual_pct"]).abs()
              for method in ("all_prior_mean", "persistence", "tal_passthrough")}
    best_baseline = min(("all_prior_mean", "persistence"), key=lambda method: errors[method].mean())
    verdict = {"mae_pp": {method: float(error.mean()) for method, error in errors.items()},
               "worst_pp": {method: float(error.max()) for method, error in errors.items()},
               "criterion": "the TAL must beat both comparators on average error and on worst year"}
    verdict["retained"] = bool(errors["tal_passthrough"].mean() < min(errors["all_prior_mean"].mean(), errors["persistence"].mean())
                               and errors["tal_passthrough"].max() <= errors[best_baseline].max())
    return table, verdict


tal_table, tal_verdict = tal_passthrough_test(leases, external)
display(tal_table.round(2))
print(tal_verdict)

{'mae_pp': {'all_prior_mean': 1.7000634438867257, 'persistence': 2.4195979786895268, 'tal_passthrough': 2.4494076936026183}, 'worst_pp': {'all_prior_mean': 2.94567593553958, 'persistence': 3.614314331499643, 'tal_passthrough': 4.299194176938575}, 'criterion': 'the TAL must beat both comparators on average error and on worst year', 'retained': False}

ANCHOR_PLAUSIBLE_RANGE_PCT = (-5.0, 25.0)   # safeguard against an erroneous anchor entry
TAL_2026_PCT = float(external_post_origin.loc[external_post_origin.metric.str.startswith("TAL 2026"), "value"].iloc[0])
ONTARIO_2026_PCT = float(external_post_origin.loc[external_post_origin.metric.eq("Ontario rent increase guideline"), "value"].iloc[0])


def regulatory_scenarios(cells, quebec_renewal_face_pct, ontario_renewal_face_pct):
    for name, anchor_pct in (("quebec", quebec_renewal_face_pct), ("ontario", ontario_renewal_face_pct)):
        if not np.isfinite(anchor_pct) or not ANCHOR_PLAUSIBLE_RANGE_PCT[0] < anchor_pct < ANCHOR_PLAUSIBLE_RANGE_PCT[1]:
            raise DataContractError(f"implausible {name} anchor: {anchor_pct}")

    def reprice(label, anchors):
        repriced = cells.copy()
        for province, anchor_pct in anchors.items():
            renewal_cells = repriced["province"].eq(province) & repriced["event_type"].eq("renewal")
            excess = (repriced.loc[renewal_cells, "face"] - anchor_pct / PERCENT).clip(lower=0.0)
            repriced.loc[renewal_cells, "effective"] -= excess
            repriced.loc[renewal_cells, "face"] -= excess
        return {"scenario": label, "effective_pct": PERCENT * aggregate(repriced), "face_pct": PERCENT * aggregate(repriced, "face")}

    return pd.DataFrame([
        reprice("Central forecast", {}),
        reprice(f"Quebec renewals at the 2026 TAL ({quebec_renewal_face_pct:g}%)", {"Quebec": quebec_renewal_face_pct}),
        reprice(f"Ontario renewals at the guideline ({ontario_renewal_face_pct:g}%), if The Met is rent-controlled",
                {"Ontario": ontario_renewal_face_pct}),
        reprice("Both anchors", {"Quebec": quebec_renewal_face_pct, "Ontario": ontario_renewal_face_pct})])


display(external_post_origin[["geography", "metric", "value", "release_date", "limitation"]])
display(regulatory_scenarios(forecast_2026["cells"], TAL_2026_PCT, ONTARIO_2026_PCT).round(3))

CONCESSION_GAP_SHOCK = 0.02     # ±2 points of concession gap on new leases
TURNOVER_SHARE_SHOCK = 0.10     # ±10 points of turnover share


def scenario_table(bundle):
    central = combine(component_cells(bundle))
    rows = []

    def add(label, cells):
        rows.append({"scenario": label, "effective_pct": PERCENT * aggregate(cells), "face_pct": PERCENT * aggregate(cells, "face")})

    add("Central forecast", central)
    for label, gap_change in (("Structural reference: current concessions", 0.0),
                              (f"Structural: concession gap +{PERCENT * CONCESSION_GAP_SHOCK:g} points", CONCESSION_GAP_SHOCK),
                              (f"Structural: concession gap −{PERCENT * CONCESSION_GAP_SHOCK:g} points", -CONCESSION_GAP_SHOCK)):
        shocked = central.copy()
        shocked["effective"] = [roster_bridge(bundle, cell.property_code, cell.face,
                                              float(np.clip(cell.new_gap_persistence + gap_change, *SPEC.gap_clip)))
                                for cell in shocked.itertuples()]
        add(label, shocked)
    for share_change in (TURNOVER_SHARE_SHOCK, -TURNOVER_SHARE_SHOCK):
        shocked = central.copy()
        for property_code in shocked.property_code.unique():
            turnover = shocked.property_code.eq(property_code) & shocked.event_type.eq("turnover")
            renewal = shocked.property_code.eq(property_code) & shocked.event_type.eq("renewal")
            turnover_share = float(np.clip(shocked.loc[turnover, "event_share"].iloc[0] + share_change, 0, 1))
            shocked.loc[turnover, "event_share"] = turnover_share
            shocked.loc[renewal, "event_share"] = 1 - turnover_share
        shocked["expected_units"] = shocked.roster_units * shocked.event_share
        shocked["weight"] = shocked.expected_units / shocked.expected_units.sum()
        add(f"Turnover share {PERCENT * share_change:+.0f} points", shocked)
    return pd.DataFrame(rows)


scenarios = scenario_table(make_bundle(leases, TARGET_YEAR))
display(scenarios.round(3))
largest_backtest_error_pp = float(all_folds.abs_error_pp.max())
historical_envelope_pct = [estimate_pct - largest_backtest_error_pp, estimate_pct + largest_backtest_error_pp]
print(f"Descriptive historical envelope: {historical_envelope_pct[0]:.2f}% to {historical_envelope_pct[1]:.2f}% "
      f"(central ± largest annual backtest error, {largest_backtest_error_pp:.2f} pts; not a confidence interval)")

Descriptive historical envelope: 2.51% to 7.10% (central ± largest annual backtest error, 2.29 pts; not a confidence interval)

BEDROOM_SHRINK_K = 20.0          # same shrinkage strength as the renewal share
BEDROOM_TRAILING_YEARS = 3       # same window as the renewal share
building_names = validated_leases.groupby("sPropCode")["sBuilding"].first().rename("building")


def building_forecast(details_cells):
    cells = details_cells.merge(building_names, left_on="property_code", right_index=True, validate="many_to_one")
    return pd.DataFrame([{"building": name, "expected_events": building_cells.expected_units.sum(),
                          "face_pct": PERCENT * np.average(building_cells.face, weights=building_cells.weight),
                          "effective_pct": PERCENT * np.average(building_cells.effective, weights=building_cells.weight)}
                         for name, building_cells in cells.groupby("building")])


def bedroom_offsets(pairs_table, target_year):
    recent = pairs_table[pairs_table["event_year"].between(target_year - BEDROOM_TRAILING_YEARS, target_year - 1)]
    if recent.empty:
        raise DataContractError("no recent pair to estimate the bedroom offsets")
    effective = clip_for_training(recent["effective_growth"], "effective_growth")
    face = clip_for_training(recent["face_growth"], "face_growth")
    rows = []
    for bedrooms, index in recent.groupby("bedrooms").groups.items():
        n_pairs = len(index)
        shrink = n_pairs / (n_pairs + BEDROOM_SHRINK_K)
        rows.append({"bedrooms": bedrooms, "n_pairs": n_pairs,
                     "effective_offset": shrink * float(effective.loc[index].mean() - effective.mean()),
                     "face_offset": shrink * float(face.loc[index].mean() - face.mean())})
    return pd.DataFrame(rows)


def bedroom_forecast(lease_table, details_cells, target_year=TARGET_YEAR):
    bundle = make_bundle(lease_table, target_year)
    offsets = bedroom_offsets(bundle.pairs, target_year).set_index("bedrooms")
    roster = bundle.roster[bundle.roster["comparable"]].merge(building_names, left_on="property_code", right_index=True,
                                                              validate="many_to_one")
    cells = details_cells.merge(building_names, left_on="property_code", right_index=True, validate="many_to_one")
    rows = []
    for building, units in roster.groupby("building"):
        building_cells = cells[cells["building"].eq(building)]
        base = {column: float(np.average(building_cells[column], weights=building_cells["weight"])) for column in ("effective", "face")}
        units_by_bedrooms = units.groupby("bedrooms").size()
        for column in ("effective", "face"):
            building_offsets = offsets[f"{column}_offset"].reindex(units_by_bedrooms.index).fillna(0.0)
            base[f"{column}_recentre"] = float(np.average(building_offsets, weights=units_by_bedrooms))
        for bedrooms, n_units in units_by_bedrooms.items():
            offset = offsets.reindex([bedrooms]).fillna(0.0).iloc[0]
            rows.append({"building": building, "bedrooms": bedrooms, "roster_units": int(n_units),
                         "effective_pct": PERCENT * (base["effective"] + offset["effective_offset"] - base["effective_recentre"]),
                         "face_pct": PERCENT * (base["face"] + offset["face_offset"] - base["face_recentre"])})
    return pd.DataFrame(rows)


def bedroom_offset_backtest(lease_table, target_years):
    # Do the bedroom offsets predict the realised spread of the following year better than "no effect"?
    rows = []
    for year in target_years:
        bundle = make_bundle(lease_table, year)
        offsets = bedroom_offsets(bundle.pairs, year).set_index("bedrooms")["effective_offset"]
        actual_pairs = realized(lease_table, year)
        realized_spread = actual_pairs.groupby("bedrooms")["effective_growth"].mean() - actual_pairs["effective_growth"].mean()
        pairs_per_group = actual_pairs.groupby("bedrooms").size()
        common = realized_spread.index.intersection(offsets.index)
        weights = pairs_per_group.loc[common]
        rows.append({"target_year": year, "bedroom_groups": len(common),
                     "mae_with_offsets_pp": PERCENT * float(np.average((offsets.loc[common] - realized_spread.loc[common]).abs(), weights=weights)),
                     "mae_no_bedroom_effect_pp": PERCENT * float(np.average(realized_spread.loc[common].abs(), weights=weights))})
    check = pd.DataFrame(rows)
    check["offsets_better"] = check["mae_with_offsets_pp"] < check["mae_no_bedroom_effect_pp"]
    return check


by_building = building_forecast(forecast_2026["cells"])
display(by_building.round(3))
by_bedroom = bedroom_forecast(leases, forecast_2026["cells"])
display(by_bedroom.round(2))
bedroom_check = bedroom_offset_backtest(leases, SPEC.evaluation_years)
display(bedroom_check.round(3))
print("Years in which the bedroom offsets beat \"no effect\":", int(bedroom_check.offsets_better.sum()), "out of", len(bedroom_check))

Years in which the bedroom offsets beat "no effect": 2 out of 6

# Dashboard: 2026 forecast by building, contractual (face) and effective, with the descriptive historical envelope.
dashboard = by_building.set_index("building")
envelope_half_width_pp = (historical_envelope_pct[1] - historical_envelope_pct[0]) / 2
rows_position = np.arange(len(dashboard))
BAR_HEIGHT = 0.4
fig, ax = plt.subplots(figsize=(9, 4))
ax.barh(rows_position + BAR_HEIGHT / 2, dashboard.face_pct, BAR_HEIGHT, label="Contractual")
ax.barh(rows_position - BAR_HEIGHT / 2, dashboard.effective_pct, BAR_HEIGHT, xerr=envelope_half_width_pp, capsize=3,
        label="Effective (± historical envelope)")
ax.set_yticks(rows_position, [f"{name} ({events:.0f} events)" for name, events in zip(dashboard.index, dashboard.expected_events)])
ax.axvline(estimate_pct, color="black", ls="--", lw=1, label=f"Effective portfolio: {estimate_pct:.2f}%")
ax.set(title="2026 forecast by building", xlabel="% annualised, same unit")
ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.18), ncol=3)
plt.tight_layout()
plt.show()

def explain_building(cells):
    # Exact reconciliation of one building: renewals + turnovers + concession adjustment (effective − contractual) = effective.
    weights = cells["weight"]
    renewal = cells["event_type"].eq("renewal")
    renewal_share = float(weights[renewal].sum() / weights.sum())
    face_renewal = float(np.average(cells.loc[renewal, "face"], weights=weights[renewal]))
    face_turnover = float(np.average(cells.loc[~renewal, "face"], weights=weights[~renewal]))
    face = float(np.average(cells["face"], weights=weights))
    effective = float(np.average(cells["effective"], weights=weights))
    return {"renewal_share": renewal_share,
            "renewal_face_pct": PERCENT * face_renewal, "turnover_face_pct": PERCENT * face_turnover,
            "renewals_pp": PERCENT * renewal_share * face_renewal,
            "turnovers_pp": PERCENT * (1 - renewal_share) * face_turnover,
            "concessions_pp": -PERCENT * (face - effective),
            "face_pct": PERCENT * face, "effective_pct": PERCENT * effective,
            "momentum_DIRECT_pct": PERCENT * float(np.average(cells["DIRECT"], weights=weights)),
            "trend_ANCHOR_pct": PERCENT * float(np.average(cells["ANCHOR"], weights=weights))}


cells_with_building = forecast_2026["cells"].merge(building_names, left_on="property_code", right_index=True, validate="many_to_one")
why = pd.DataFrame({building: explain_building(cells) for building, cells in cells_with_building.groupby("building")}).T
pairs_with_building = eligible_pairs.merge(building_names, left_on="property_code", right_index=True, validate="many_to_one")
why["realised_2025_effective_pct"] = PERCENT * pairs_with_building[pairs_with_building.event_year.eq(LAST_OBSERVED_YEAR)].groupby(
    "building").effective_growth.mean()
why = why.sort_values("effective_pct", ascending=False)

# The three pieces land exactly on the by-building forecast of section 8 (an arithmetic check, not a causal attribution).
np.testing.assert_allclose(why.renewals_pp + why.turnovers_pp + why.concessions_pp, why.effective_pct, atol=1e-9)
np.testing.assert_allclose(why.effective_pct, by_building.set_index("building").loc[why.index, "effective_pct"], atol=1e-9)
display(why.round(2))

WATERFALL_STEPS = [("renewals_pp", "Renewals", "tab:blue"), ("turnovers_pp", "Turnovers", "tab:cyan"),
                   ("concessions_pp", "Concession\nadjustment", "tab:red")]
fig, axes = plt.subplots(2, 3, figsize=(14, 7), sharey=True)
for ax, (building, row) in zip(axes.ravel(), why.iterrows()):
    level = 0.0
    for position, (column, label, color) in enumerate(WATERFALL_STEPS):
        ax.bar(position, row[column], bottom=level, color=color)
        ax.text(position, level + row[column] / 2, f"{row[column]:+.1f}", ha="center", va="center", color="white", fontweight="bold")
        level += row[column]
    ax.bar(len(WATERFALL_STEPS), row.effective_pct, color="black")
    ax.text(len(WATERFALL_STEPS), row.effective_pct / 2, f"{row.effective_pct:.1f}%", ha="center", va="center", color="white", fontweight="bold")
    ax.set_xticks(range(len(WATERFALL_STEPS) + 1), [label for _, label, _ in WATERFALL_STEPS] + ["Effective\n2026"])
    ax.set_title(building)
    ax.axhline(0, color="grey", lw=0.8)
axes[0, 0].set_ylabel("percentage points")
axes[1, 0].set_ylabel("percentage points")
fig.suptitle("What makes up each building's 2026 increase", fontsize=13)
plt.tight_layout()
plt.show()

from IPython.display import Markdown

FLOOD_ZONE_NEAR_M = 250          # below this distance, proximity to a flood zone is worth flagging
building_context = pd.read_csv("building_context.csv").set_index("building")


def en_number(value, decimals=1):
    return f"{value:,.{decimals}f}"


def building_story(building, row, context):
    lines = [f"**{building}: {en_number(row.effective_pct)}% effective** ({en_number(row.face_pct)}% contractual).",
             f"On paper, rents rise by {en_number(row.face_pct)}%: about {en_number(PERCENT * row.renewal_share, 0)}% of the expected leases "
             f"are renewals (forecast at +{en_number(row.renewal_face_pct)}%) and the rest turnovers (+{en_number(row.turnover_face_pct)}%). "
             f"The concession adjustment (the gap between the effective and contractual forecasts) takes back {en_number(-row.concessions_pp)} {'point' if en_number(-row.concessions_pp) == '1.0' else 'points'}.",
             f"Reference points: in {LAST_OBSERVED_YEAR}, its same-unit effective rents rose by {en_number(row.realised_2025_effective_pct)}%; "
             f"its long-term trend is {en_number(row.trend_ANCHOR_pct)}% a year."]
    notes = [f"{en_number(context.rapid_transit_distance_m, 0)} m from {context.nearest_rapid_transit}",
             f"location score {context.location_score_0_100}/100"]
    if pd.notna(context.units_permitted_2km_2023_2025):
        notes.append(f"{en_number(context.units_permitted_2km_2023_2025, 0)} units permitted within 2 km in 2023–2025 "
                     "(new supply to watch: more supply tends to increase concessions)")
    else:
        notes.append("building permits not published by the city")
    if context.flood_zone_distance_m < FLOOD_ZONE_NEAR_M:
        notes.append(f"{context.flood_zone_distance_m} m from a mapped flood zone")
    if context.heat_island is True or str(context.heat_island) == "True":
        notes.append("urban heat island")
    lines.append("Public context (not used in the calculation): " + "; ".join(notes) + ".")
    return "\n\n".join(lines)


display(Markdown("\n\n---\n\n".join(building_story(building, row, building_context.loc[building]) for building, row in why.iterrows())))

from sklearn.linear_model import Ridge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

RANDOM_SEED = 20_260_101
XGB_PARAMS = dict(max_depth=3, n_estimators=300, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8,
                  min_child_weight=10, reg_lambda=1.0, random_state=RANDOM_SEED, n_jobs=1)
RIDGE_ALPHA = 10.0
LAG_FEATURES = ["lag_cell_face", "lag_cell_eff", "lag_cell_gap", "lag_port_face", "lag_port_gap"]
MODEL_FEATURES = ["is_renewal", "is_ontario", "bedrooms", "interval_years", "prior_gap", "start_month",
                  "property_age", "rent_position"] + LAG_FEATURES


def lagged_means(known_pairs):
    # Previous year's averages, by province × event and for the portfolio.
    cell = known_pairs.groupby(["event_year", "province", "event_type"]).agg(
        lag_cell_face=("face_growth", "mean"), lag_cell_eff=("effective_growth", "mean"), lag_cell_gap=("new_gap", "mean")).reset_index()
    portfolio = known_pairs.groupby("event_year").agg(lag_port_face=("face_growth", "mean"), lag_port_gap=("new_gap", "mean")).reset_index()
    cell["event_year"] += 1
    portfolio["event_year"] += 1
    return cell, portfolio


def model_features(rows, known_pairs):
    cell, portfolio = lagged_means(known_pairs)
    first_year = known_pairs.groupby("property_code")["event_year"].min()
    mean_prior_rent = known_pairs.groupby("property_code")["prior_face_rent"].mean()
    features = rows.merge(cell, on=["event_year", "province", "event_type"], how="left").merge(portfolio, on="event_year", how="left")
    features["is_renewal"] = features["event_type"].eq("renewal").astype(float)
    features["is_ontario"] = features["province"].eq("Ontario").astype(float)
    features["property_age"] = features["event_year"] - features["property_code"].map(first_year).fillna(features["event_year"])
    features["rent_position"] = features["prior_face_rent"] / features["property_code"].map(mean_prior_rent).fillna(features["prior_face_rent"])
    features["bedrooms"] = features["bedrooms"].astype(float)
    return features


def roster_rows(bundle):
    # One row per roster unit and per event type, weighted by the expected share.
    roster = bundle.roster[bundle.roster["comparable"]].reset_index(drop=True)
    shares = event_mix(bundle).set_index(["property_code", "event_type"])["event_share"]
    return pd.concat([pd.DataFrame({"unit": roster.index, "event_year": bundle.target_year, "province": roster["province"],
                                    "event_type": event_type, "property_code": roster["property_code"],
                                    "bedrooms": roster["bedrooms"], "interval_years": roster["expected_years"],
                                    "prior_gap": roster["prior_gap"],
                                    "start_month": (roster["lease_end"] + pd.Timedelta(days=1)).dt.month,
                                    "prior_face_rent": roster["face_rent"],
                                    "share": [shares[(code, event_type)] for code in roster["property_code"]]})
                      for event_type in ("renewal", "turnover")], ignore_index=True)


def fit_predict(kind, train, test):
    target = clip_for_training(train["effective_growth"], "effective_growth").to_numpy()
    if kind == "xgboost":
        regressor = xgboost.XGBRegressor(**XGB_PARAMS)
        regressor.fit(train[MODEL_FEATURES].to_numpy(float), target)
        return regressor.predict(test[MODEL_FEATURES].to_numpy(float)), regressor
    has_lags = train[LAG_FEATURES].notna().all(axis=1)
    usable = [feature for feature in MODEL_FEATURES if train.loc[has_lags, feature].notna().any()]
    training_means = train.loc[has_lags, usable].mean()
    regressor = make_pipeline(StandardScaler(), Ridge(alpha=RIDGE_ALPHA))
    regressor.fit(train.loc[has_lags, usable].fillna(training_means).to_numpy(float), target[has_lags.to_numpy()])
    return regressor.predict(test[usable].fillna(training_means).to_numpy(float)), regressor


def forecast_challenger(bundle, kind):
    known_pairs = bundle.pairs
    train = model_features(known_pairs.assign(start_month=known_pairs["lease_start"].dt.month), known_pairs)
    roster = model_features(roster_rows(bundle), known_pairs)
    predictions, regressor = fit_predict(kind, train, roster)
    roster = roster.assign(prediction=predictions)
    per_unit = roster.groupby("unit").apply(lambda unit: float(np.average(unit["prediction"], weights=unit["share"])),
                                            include_groups=False)
    return {"portfolio": float(per_unit.mean()), "roster": roster, "model": regressor}


challenger_rows = []
for year in evaluation_years:
    bundle = make_bundle(leases, year)
    challenger_rows.append({"target_year": year,
                            "actual_pct": PERCENT * float(realized(leases, year)["effective_growth"].mean()),
                            "our_method_pct": all_folds.set_index("target_year").loc[year, "predicted_pct"],
                            "xgboost_pct": PERCENT * forecast_challenger(bundle, "xgboost")["portfolio"],
                            "ridge_pct": PERCENT * forecast_challenger(bundle, "ridge")["portfolio"],
                            "historical_mean_pct": baselines.query("model == 'All history pooled' and target_year == @year").predicted_pct.iloc[0]})
challenger_backtest = pd.DataFrame(challenger_rows)
display(challenger_backtest.round(2))

scoreboard_rows = []
for method in ("our_method", "xgboost", "ridge", "historical_mean"):
    error = challenger_backtest[f"{method}_pct"] - challenger_backtest["actual_pct"]
    in_required_window = challenger_backtest["target_year"].isin(REQUIRED_BACKTEST_YEARS)
    scoreboard_rows.append({"method": method, "MAE_2020_2025_pp": error.abs().mean(),
                            "MAE_2023_2025_pp": error[in_required_window].abs().mean(), "worst_year_pp": error.abs().max()})
challenger_scoreboard = pd.DataFrame(scoreboard_rows).set_index("method")
display(challenger_scoreboard.round(3))
for candidate in ("xgboost", "ridge"):
    board = challenger_scoreboard
    replaces = bool(board.loc[candidate, "MAE_2020_2025_pp"] < board.loc[["our_method", "historical_mean"], "MAE_2020_2025_pp"].min()
                    and board.loc[candidate, "MAE_2023_2025_pp"] < board.loc[["our_method", "historical_mean"], "MAE_2023_2025_pp"].min()
                    and board.loc[candidate, "worst_year_pp"] <= board.loc["our_method", "worst_year_pp"])
    print(f"{candidate} replaces our method: {replaces}")

xgboost replaces our method: False
ridge replaces our method: False

# Why XGBoost forecasts this figure for 2026: SHAP contributions (TreeSHAP) weighted like the forecast.
TOP_FEATURES_SHOWN = 10
xgb_2026 = forecast_challenger(make_bundle(leases, TARGET_YEAR), "xgboost")
roster_2026 = xgb_2026["roster"]
contributions = xgb_2026["model"].get_booster().predict(
    xgboost.DMatrix(roster_2026[MODEL_FEATURES].to_numpy(float), feature_names=MODEL_FEATURES), pred_contribs=True)
row_weights = roster_2026["share"].to_numpy() / roster_2026["unit"].nunique()
shap_2026 = (pd.DataFrame({"feature": MODEL_FEATURES,
                           "contribution_to_2026_pp": PERCENT * (contributions[:, :-1] * row_weights[:, None]).sum(axis=0),
                           "mean_abs_pp": PERCENT * np.abs(contributions[:, :-1]).mean(axis=0)})
             .sort_values("mean_abs_pp", ascending=False).reset_index(drop=True))
display(shap_2026.round(3))
top_features = shap_2026.head(TOP_FEATURES_SHOWN).iloc[::-1]
plt.figure(figsize=(8, 4))
plt.barh(top_features.feature, top_features.contribution_to_2026_pp)
plt.title("XGBoost 2026: mean SHAP contribution by variable (points)")
plt.xlabel("percentage points")
plt.tight_layout()
plt.show()
print(f"2026 — XGBoost: {PERCENT * xgb_2026['portfolio']:.2f}% | our method: {estimate_pct:.2f}%")

2026 — XGBoost: 5.09% | our method: 4.80%

Grading criterion	Section
Definition of the increase (10)	0 and 7.1
Data analysis and mix effect (15)	1b, 1c, 2, 3
Same-unit matching (15)	4
Concessions (10)	5
Renewals vs turnovers, Ontario (10)	6, 7.6
External data (15)	7.2, 7.6
Forecast and backtest (15)	7.3 to 7.7, 9
Bonus: building × bedrooms, scenarios, dashboard	7.7, 8, 8b
Beyond the brief (optional prototype)	10

Possible definition	Why we do not use it as the headline figure
Annual median of the whole portfolio	Mostly measures mix: the arrival of three upscale buildings creates a 10.8% "increase" in 2023 that did not happen (section 3).
Same unit, contractual (face) rent (`sRent`)	Ignores free months. Since 2024 the gap between contractual (face) and collected rent has been widening (section 5): contractual (face) rent overstates what Équinoxe collects. Shown alongside (6.38%).
Renewals only / turnovers only	Useful for setting rents, but a revenue budget needs both. The two are separated in sections 6 and 7.3 and recombined by their expected share.
Total revenue or NOI	Impossible with this extract: no occupancy or vacancy data (section 2).

	type	distinct_values
hProperty	int64	9
hUnit	int64	1061
hBuilding	int64	6
sPropCode	object	9
sUnitCode	object	519
sBuilding	object	6
sCity	object	4
sState	object	2
sAddr1	object	6
sLatitude	float64	6
sLongitude	float64	6
sLeaseTerm	object	4
sPropType	object	1
sRent	int64	666
sBeds	int64	4
sBaths	float64	3
sSqft	int64	199
sLink	object	1061
sFurnishing	object	1
sAvailable	datetime64[ns]	1302
sSmoking	object	1
sCats	bool	1
sDogs	bool	1
sRentEffective	float64	3178
sConcession	int64	2
sSite	object	6
sUnitSubtype	object	4
sUnitType	object	11
sSignDate	datetime64[ns]	2067
sLeaseFrom	datetime64[ns]	1302
sLeaseTo	datetime64[ns]	1341
sTermMonths	float64	4
sTermSeq	int64	8
sRenewal	int64	2
sFloor	int64	27
year	int32	9

`sChargeCode`	What it is	Classification
`PromoPay`, `freerent`	Discount on the rent	Rent
`freepark`, `freelock`, `freeappl`	Free parking, locker, appliances	Service (not rent)
`Freeothe`, `Indem`, `Referenc`	Other, indemnity, referral	Other, kept separate

	leases	median_rent	yoy_pct
year
2019	328	1730.0	NaN
2020	373	1775.0	2.60
2021	361	1850.0	4.23
2022	422	1912.5	3.38
2023	693	2120.0	10.85
2024	902	2215.0	4.48
2025	958	2305.0	4.06

Équinoxe Collection — 2026 rent increase¶

One-page summary¶

0. The question we are answering¶

1. Load the data¶

1b. Reading the columns: `h` fields and `s` fields¶

1c. What the columns really represent, and how we rename them¶

2. What we are looking at¶

3. The obvious approach, and why it misleads¶

4. The method that works: comparing a unit with itself¶

4b. Our matching: rules fixed once, exclusions visible¶

5. Contractual (face) rent is not collected rent¶

5b. Checking `sRentEffective` against the concession ledger¶

5c. Why contractual (face) and effective growth diverge from 2024¶

6. Renewals do not behave like turnovers¶

7. Your turn: our forecast¶

7.1 The question (recap)¶

7.2 Public data and the date rule¶

7.3 Method: three simple components, forecast by building × event type¶

7.4 `estimate_2026()` and `backtest()` completed¶

7.5 Backtest on 2023, 2024 and 2025, and comparison with simple methods¶

7.6 Reconciliation with public data¶

7.7 Scenarios and historical envelope¶

8. Bonus: forecast by building and by number of bedrooms, small dashboard¶

8b. Why this figure? A building-by-building reading¶

9. Does a regression model do better? XGBoost and Ridge, tested and rejected¶

10. Beyond the brief: what we built in addition¶

11. Assumptions, limitations, and how to improve without adding noise¶

References¶

	new name	what the column represents
original column
sPropCode	property_code	property code (one phase of a building); half ...
sUnitCode	unit_code	unit number within the property; the other hal...
sState	province	province: Quebec (TAL) or Ontario (guideline)
sBeds	bedrooms	number of bedrooms
sSignDate	sign_date	signature date: used to know what was known at...
sLeaseFrom	lease_start	lease start: date of the measured event
sLeaseTo	lease_end	contractual lease end: used to anticipate the ...
sTermSeq	term_seq	rank of the lease in the unit's history: defin...
sRent	face_rent	monthly contractual (face) rent, the rent writ...
sRentEffective	effective_rent	monthly effective rent, after concessions amor...

	leases	median_sqft	pct_2bed_plus	buildings	rent_per_sqft
year
2019	328	1177.5	78.66	3	1.48
2020	373	1180.0	77.75	3	1.53
2021	361	1175.0	78.39	3	1.60
2022	422	1160.0	72.51	3	1.71
2023	693	1095.0	62.63	5	2.04
2024	902	1045.0	62.20	6	2.20
2025	958	1025.0	60.23	6	2.33

	event_year	pairs	face_pct	effective_pct	median_effective_pct	renewal_share	naive_median_face_change_pct
0	2018	88	2.126	2.041	1.976	0.534	-11.295
1	2019	159	2.000	1.593	1.629	0.579	7.453
2	2020	331	3.034	2.769	3.019	0.604	2.601
3	2021	355	4.384	3.753	3.543	0.620	4.225
4	2022	352	5.273	4.716	4.586	0.642	3.378
5	2023	416	2.089	2.178	2.019	0.575	10.850
6	2024	698	3.728	1.951	1.772	0.605	4.481
7	2025	780	7.559	3.833	3.575	0.606	4.063

	leases	pct_with_concession	median_contracted	median_effective	gap_pct
year
2021	361	54.02	1850.0	1785.00	-3.51
2022	422	57.35	1912.5	1835.00	-4.05
2023	693	54.26	2120.0	2025.40	-4.46
2024	902	73.06	2215.0	2084.46	-5.89
2025	958	85.39	2305.0	2105.59	-8.65

	contracted	effective
year
2021	4.46	3.54
2022	5.40	4.59
2023	2.29	2.02
2024	3.39	1.77
2025	7.04	3.58

	event_year	n	face_pct	rent_only_pct	supplied_effective_pct
0	2018	84	2.038	1.893	2.003
1	2019	142	2.034	1.361	1.740
2	2020	280	2.998	3.116	2.843
3	2021	283	4.503	3.717	3.813
4	2022	273	5.252	5.689	4.641
5	2023	336	2.078	1.693	2.003
6	2024	534	3.761	2.551	2.232
7	2025	502	7.542	5.942	3.775

	event_year	pairs	face_pct	effective_pct	face_minus_effective_pp	prior_gap_pct	new_gap_pct	gap_change_pp	incidence_prior	incidence_new	depth_prior_pct	depth_new_pct	gap_change_from_incidence_pp	gap_change_from_depth_pp
0	2018	88	2.13	2.04	0.08	1.21	1.36	0.15	0.40	0.38	3.04	3.62	-0.08	0.22
1	2019	159	2.00	1.59	0.41	1.36	1.78	0.42	0.38	0.47	3.54	3.77	0.32	0.10
2	2020	331	3.03	2.77	0.27	1.77	2.07	0.30	0.46	0.48	3.84	4.28	0.10	0.20
3	2021	355	4.38	3.75	0.63	2.02	2.68	0.66	0.47	0.54	4.29	4.99	0.31	0.35
4	2022	352	5.27	4.72	0.56	2.74	3.35	0.61	0.56	0.58	4.92	5.78	0.12	0.49
5	2023	416	2.09	2.18	-0.09	3.21	3.24	0.03	0.55	0.49	5.81	6.64	-0.40	0.43
6	2024	698	3.73	1.95	1.78	3.66	5.46	1.79	0.55	0.70	6.63	7.79	1.06	0.73
7	2025	780	7.56	3.83	3.73	5.79	9.24	3.46	0.74	0.86	7.79	10.70	1.13	2.33

	lease_start_year	freeappl	freelock	freeothe	freepark	freerent	indem	promopay	referenc	total_matched_pct	supplied_gap_pct	unexplained_by_ledger_pct
0	2017	0.00	0.12	0.19	0.49	0.03	0.00	0.64	0.00	1.47	1.18	-0.29
1	2018	0.07	0.17	0.22	0.44	0.07	0.03	0.73	0.06	1.79	1.36	-0.43
2	2019	0.00	0.12	0.37	0.65	0.09	0.05	1.21	0.00	2.50	1.70	-0.80
3	2020	0.03	0.11	0.41	0.47	0.09	0.04	1.08	0.03	2.25	2.09	-0.16
4	2021	0.06	0.09	0.43	0.80	0.11	0.04	1.82	0.00	3.35	2.69	-0.66
5	2022	0.02	0.13	0.55	0.96	0.12	0.05	1.58	0.06	3.47	3.33	-0.15
6	2023	0.04	0.27	0.82	1.72	0.30	0.11	1.77	0.05	5.09	3.58	-1.51
7	2024	0.09	0.34	1.25	1.67	0.33	0.10	2.56	0.06	6.39	5.71	-0.69
8	2025	0.07	0.27	0.77	1.62	0.19	0.10	3.77	0.06	6.83	9.13	2.30

	pairs_turnover	pairs_renewal	median_turnover	median_renewal
year
2022	126	226	4.22	5.69
2023	177	239	1.85	2.58
2024	276	422	4.85	2.94
2025	307	473	8.98	6.41

	event_year	province	event_type	n	face_pct	effective_pct
10	2023	Quebec	renewal	239	2.550	2.747
11	2023	Quebec	turnover	177	1.467	1.409
12	2024	Ontario	renewal	67	2.899	0.328
13	2024	Ontario	turnover	41	5.080	4.756
14	2024	Quebec	renewal	355	2.915	1.368
15	2024	Quebec	turnover	235	4.956	2.803
16	2025	Ontario	renewal	63	6.333	1.893
17	2025	Ontario	turnover	54	8.925	4.547
18	2025	Quebec	renewal	410	6.529	2.603
19	2025	Quebec	turnover	253	9.243	6.157

	geography	metric	reference_period	release_date	value	unit
3	Quebec	TAL estimated average basic adjustment (lagged)	TAL 2025	2025-01-21	5.9000	percent
7	Ontario	Ontario rent increase guideline	2026 guideline	2025-06-30	2.1000	percent
11	Montreal CMA	CMHC fixed-sample average rent change	October 2025	2025-12-11	7.4000	percent
15	Ottawa-Gatineau CMA (Ontario part)	CMHC fixed-sample average rent change	October 2025	2025-12-11	3.5000	percent
19	Montreal CMA	CMHC apartment vacancy rate	October 2025	2025-12-11	2.9000	percent
23	Ottawa-Gatineau CMA (Ontario part)	CMHC apartment vacancy rate	October 2025	2025-12-11	3.0000	percent
27	Montreal CMA	Statistics Canada rented accommodation CPI yea...	November 2025	2025-12-15	7.3678	percent YoY
31	Ontario	Statistics Canada rent CPI year-over-year	November 2025	2025-12-15	4.0687	percent YoY

	target_year	predicted_pct	actual_pct	error_pp
0	2023	4.472	2.178	2.294
1	2024	2.620	1.951	0.669
2	2025	2.928	3.833	-0.905

	target_year	predicted_pct	actual_pct	error_pp	predicted_face_pct	actual_face_pct	face_error_pp	segment_mae_pp	roster_units	realized_pairs
0	2020	1.764	2.769	-1.005	2.005	3.034	-1.029	0.957	306	331
1	2021	2.743	3.753	-1.010	2.875	4.384	-1.509	1.149	353	355
2	2022	3.747	4.716	-0.968	3.881	5.273	-1.392	0.955	347	352
3	2023	4.472	2.178	2.294	4.581	2.089	2.492	2.248	402	416
4	2024	2.620	1.951	0.669	2.755	3.728	-0.973	1.743	640	698
5	2025	2.928	3.833	-0.905	3.589	7.559	-3.970	1.314	845	780

	MAE_pp	bias_pp	worst_pp	MAE_2023_2025_pp
model
All history pooled	1.282	-0.503	1.841	1.138
Last year pooled	1.295	-0.373	2.538	1.549
Trailing 3 years pooled	1.409	-0.372	1.754	1.421
Our method (fixed)	1.142	-0.154	2.294	1.289

	target_year	realized_in_roster	target_roster_coverage	roster_observed_fraction	segment_target_coverage
0	2020	303	0.915	0.990	1.000
1	2021	347	0.977	0.983	1.000
2	2022	342	0.972	0.986	1.000
3	2023	398	0.957	0.990	0.998
4	2024	629	0.901	0.983	0.990
5	2025	763	0.978	0.903	1.000

	year	actual_pct	all_prior_mean_pct	persistence_pct	tal_passthrough_pct	tal_pct
0	2023	2.55	3.92	5.83	6.85	2.3
1	2024	2.91	3.69	2.55	4.25	4.0
2	2025	6.53	3.58	2.91	4.81	5.9

	geography	metric	value	release_date	limitation
0	Quebec	TAL 2026 base rent-adjustment percentage	3.10	2026-01	Published AFTER the 2025-12-31 forecast origin...
1	Ontario	Ontario rent increase guideline	2.10	2025-06-30	Applies only to rent-controlled units; buildin...
2	Quebec	Statistics Canada rent CPI year-over-year, Aug...	3.02	2026-09-14	Single monthly reading; monthly rent CPI is no...
3	Ontario	Statistics Canada rent CPI year-over-year, Aug...	2.37	2026-09-14	Single monthly reading; not used in any estimate.

	scenario	effective_pct	face_pct
0	Central forecast	4.805	6.378
1	Quebec renewals at the 2026 TAL (3.1%)	3.465	5.038
2	Ontario renewals at the guideline (2.1%), if T...	4.557	6.130
3	Both anchors	3.217	4.790

	building	expected_events	face_pct	effective_pct
0	Daniel-Johnson	128.0	6.055	4.937
1	Le Carlyle	180.0	6.428	4.522
2	Levesque	75.0	5.745	4.749
3	Saint-Elzear	267.0	6.017	4.967
4	The Met	119.0	6.566	4.317
5	Westpark	159.0	7.347	5.137

	building	bedrooms	roster_units	effective_pct	face_pct
0	Daniel-Johnson	0	12	4.98	6.34
1	Daniel-Johnson	1	54	4.94	6.24
2	Daniel-Johnson	2	58	4.94	5.82
3	Daniel-Johnson	3	4	4.71	6.09
4	Le Carlyle	1	68	4.53	6.69
5	Le Carlyle	2	107	4.53	6.26
6	Le Carlyle	3	5	4.30	6.53
7	Levesque	1	9	4.76	6.11
8	Levesque	2	64	4.75	5.69
9	Levesque	3	2	4.53	5.96
10	Saint-Elzear	1	77	4.98	6.31
11	Saint-Elzear	2	182	4.97	5.89
12	Saint-Elzear	3	8	4.75	6.16
13	The Met	0	37	4.34	6.76
14	The Met	1	46	4.31	6.67
15	The Met	2	36	4.30	6.24
16	Westpark	1	61	5.15	7.61
17	Westpark	2	95	5.14	7.18
18	Westpark	3	3	4.91	7.45

	target_year	bedroom_groups	mae_with_offsets_pp	mae_no_bedroom_effect_pp	offsets_better
0	2020	3	0.183	0.108	False
1	2021	3	0.071	0.073	True
2	2022	3	0.212	0.256	True
3	2023	3	0.162	0.087	False
4	2024	4	0.349	0.194	False
5	2025	4	0.124	0.122	False

	renewal_share	renewal_face_pct	turnover_face_pct	renewals_pp	turnovers_pp	concessions_pp	face_pct	effective_pct	momentum_DIRECT_pct	trend_ANCHOR_pct	realised_2025_effective_pct
Westpark	0.62	6.14	9.34	3.83	3.52	-2.21	7.35	5.14	3.80	3.81	3.78
Saint-Elzear	0.60	5.47	6.84	3.29	2.73	-1.05	6.02	4.97	4.39	3.17	4.64
Daniel-Johnson	0.60	5.20	7.33	3.11	2.94	-1.12	6.05	4.94	4.14	2.96	4.41
Levesque	0.55	5.38	6.19	2.96	2.78	-1.00	5.75	4.75	3.59	2.89	3.17
Le Carlyle	0.63	5.80	7.50	3.65	2.78	-1.91	6.43	4.52	3.59	2.55	3.54
The Met	0.58	5.45	8.10	3.15	3.42	-2.25	6.57	4.32	3.01	2.59	3.12

	target_year	actual_pct	our_method_pct	xgboost_pct	ridge_pct	historical_mean_pct
0	2020	2.77	1.76	2.05	2.22	1.75
1	2021	3.75	2.74	2.83	1.63	2.33
2	2022	4.72	3.75	4.52	5.87	2.87
3	2023	2.18	4.47	4.82	6.52	3.38
4	2024	1.95	2.62	3.16	-0.15	3.09
5	2025	3.83	2.93	4.61	5.53	2.76

	MAE_2020_2025_pp	MAE_2023_2025_pp	worst_year_pp
method
our_method	1.142	1.289	2.294
xgboost	1.075	1.540	2.639
ridge	1.994	2.714	4.343
historical_mean	1.282	1.138	1.841

Équinoxe Collection — 2026 rent increase¶

One-page summary¶

0. The question we are answering¶

1. Load the data¶

1b. Reading the columns: h fields and s fields¶

1c. What the columns really represent, and how we rename them¶

2. What we are looking at¶

3. The obvious approach, and why it misleads¶

4. The method that works: comparing a unit with itself¶

4b. Our matching: rules fixed once, exclusions visible¶

5. Contractual (face) rent is not collected rent¶

5b. Checking sRentEffective against the concession ledger¶

5c. Why contractual (face) and effective growth diverge from 2024¶

6. Renewals do not behave like turnovers¶

7. Your turn: our forecast¶

7.1 The question (recap)¶

7.2 Public data and the date rule¶

7.3 Method: three simple components, forecast by building × event type¶

7.4 estimate_2026() and backtest() completed¶

7.5 Backtest on 2023, 2024 and 2025, and comparison with simple methods¶

7.6 Reconciliation with public data¶

7.7 Scenarios and historical envelope¶

8. Bonus: forecast by building and by number of bedrooms, small dashboard¶

8b. Why this figure? A building-by-building reading¶

9. Does a regression model do better? XGBoost and Ridge, tested and rejected¶

10. Beyond the brief: what we built in addition¶

11. Assumptions, limitations, and how to improve without adding noise¶

References¶

1b. Reading the columns: `h` fields and `s` fields¶

5b. Checking `sRentEffective` against the concession ledger¶

7.4 `estimate_2026()` and `backtest()` completed¶

	feature	contribution_to_2026_pp	mean_abs_pp
0	prior_gap	2.509	3.546
1	lag_port_gap	-0.417	0.416
2	property_age	-0.009	0.332
3	lag_port_face	-0.303	0.305
4	lag_cell_face	0.301	0.302
5	lag_cell_gap	-0.260	0.267
6	interval_years	0.150	0.233
7	rent_position	0.010	0.207
8	start_month	-0.006	0.177
9	is_renewal	0.004	0.174
10	lag_cell_eff	0.108	0.109
11	is_ontario	-0.021	0.053
12	bedrooms	-0.007	0.046