--- title: Calibration keywords: fastai sidebar: home_sidebar summary: "Functionality to calibrate a trained, binary classification model using temperature scaling." description: "Functionality to calibrate a trained, binary classification model using temperature scaling." nb_path: "nbs/052c_calibration.ipynb" ---
{% raw %}
{% endraw %} {% raw %}
from tsai.data.external import get_UCR_data
from tsai.data.preprocessing import TSRobustScale
from tsai.learner import ts_learner
from tsai.models.FCNPlus import FCNPlus
{% endraw %} {% raw %}
{% endraw %} {% raw %}

class ModelWithTemperature[source]

ModelWithTemperature(model) :: Module

A decorator which wraps a model with temperature scaling

{% endraw %} {% raw %}

class TemperatureSetter[source]

TemperatureSetter(model, lr=0.01, max_iter=1000, line_search_fn=None, n_bins=10, verbose=True) :: Module

Calibrates a binary classification model optimizing temperature

{% endraw %} {% raw %}

class ECELoss[source]

ECELoss(n_bins=10) :: Module

Calculates the Expected Calibration Error of a model.

{% endraw %} {% raw %}
{% endraw %} {% raw %}

plot_calibration_curve[source]

plot_calibration_curve(labels, logits, cal_logits=None, figsize=(6, 6), n_bins=10, strategy='uniform')

{% endraw %} {% raw %}
{% endraw %} {% raw %}

Learner.calibrate_model[source]

Learner.calibrate_model(X=None, y=None, lr=0.01, max_iter=10000, line_search_fn=None, n_bins=10, strategy='uniform', show_plot=True, figsize=(6, 6), verbose=True)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
from tsai.data.external import get_UCR_data
from tsai.data.preprocessing import TSRobustScale
from tsai.learner import ts_learner
from tsai.models.FCNPlus import FCNPlus

from tsai.data.external import get_UCR_data
from tsai.data.preprocessing import TSRobustScale
from tsai.data.core import TSClassification, get_ts_dls
from tsai.learner import ts_learner
from tsai.models.FCNPlus import FCNPlus
from fastai.metrics import accuracy

X, y, splits = get_UCR_data('FingerMovements', split_data=False)
tfms  = [None, [TSClassification()]]
batch_tfms = TSRobustScale()
dls = get_ts_dls(X, y, splits=splits, tfms=tfms, batch_tfms=batch_tfms)
learn = ts_learner(dls, FCNPlus, metrics=accuracy)
learn.fit_one_cycle(2)
epoch train_loss valid_loss accuracy time
0 0.722970 0.700500 0.520000 00:03
1 0.693184 0.694762 0.480000 00:02
{% endraw %} {% raw %}
learn.calibrate_model()
calibrated_model = learn.calibrated_model
Before temperature - NLL: 0.695, ECE: 0.056
Calibrating the model...
...model calibrated
Optimal temperature: 1.000
After temperature  - NLL: 0.695, ECE: 0.056

{% endraw %}