---
title: NumPy Evaluation
keywords: fastai
sidebar: home_sidebar
summary: "The most important evaluation signal is the forecast error, which is the difference between the observed value $y_{\\tau}$ and the prediction $\\hat{y}_{\\tau}$, at time $\\tau$: $$e_{\\tau} = y_{\\tau}-\\hat{y}_{\\tau} \\qquad \\qquad \\tau \\in \\{t+1,\\dots,t+H \\}.$$The forecast accuracy summarizes the forecast errors in different metrics:
1. Scale-dependent errors - These metrics are on the same scale as the data.
import unittest
import torch as t
import numpy as np
from neuralforecast.losses.pytorch import (
MAELoss, MSELoss, RMSELoss, # unscaled errors
MAPELoss, SMAPELoss, # percentage errors
MASELoss, # scaled error
QuantileLoss, MQLoss # probabilistic errors
)
from neuralforecast.losses.numpy import (
mae, mse, rmse, # unscaled errors
mape, smape, # percentage errors
mase, # scaled error
quantile_loss, mqloss # probabilistic errors
)
class TestLoss(unittest.TestCase):
def setUp(self):
np.random.seed(0)
n_qs = np.random.randint(3, 10)
a = np.random.randint(1, 300)
b = np.random.randint(1, 300)
self.y = t.rand(a, b)
self.y_hat = t.rand(a, b)
self.yq_hat = t.rand(a, b, n_qs)
# Uniform from [0.0, 1.0)
self.q = np.random.random_sample()
self.qs = t.rand(n_qs)
def test_mae(self):
mae_numpy = mae(self.y, self.y_hat)
mae_pytorch = MAELoss(self.y, self.y_hat).numpy()
self.assertAlmostEqual(mae_numpy, mae_pytorch, places=6)
def test_mse(self):
mse_numpy = mse(self.y, self.y_hat)
mse_pytorch = MSELoss(self.y, self.y_hat).numpy()
self.assertAlmostEqual(mse_numpy, mse_pytorch, places=6)
def test_rmse(self):
rmse_numpy = rmse(self.y, self.y_hat)
rmse_pytorch = RMSELoss(self.y, self.y_hat).numpy()
self.assertAlmostEqual(rmse_numpy, rmse_pytorch, places=6)
def test_mape(self):
mape_numpy = mae(self.y, self.y_hat)
mape_pytorch = MAELoss(self.y, self.y_hat).numpy()
self.assertAlmostEqual(mape_numpy, mape_pytorch, places=6)
def test_smape(self):
smape_numpy = smape(self.y, self.y_hat)
smape_pytorch = SMAPELoss(self.y, self.y_hat).numpy()
self.assertAlmostEqual(smape_numpy, smape_pytorch, places=4)
'''
def test_mase(self):
y_insample = t.rand(self.first_num, self.second_num)
seasonality = 24
# Hourly 24, Daily 7, Weekly 52
# Monthly 12, Quarterly 4, Yearly 1
mase_numpy = mase(self.y, self.y_hat, y_insample, seasonality)
mase_pytorch = MASELoss(self.y, self.y_hat, y_insample, seasonality).numpy()
self.assertAlmostEqual(mase_numpy, mase_pytorch, places=5)
'''
'''
def test_rmae(self):
first_num = random.randint(0, 300)
second_num = random.randint(0, 300)
y = t.rand(first_num, second_num)
y_hat = t.rand(first_num, second_num)
rmae_result_pytorch = pytorch_loss.RMAELoss(y, y_hat, seasonality)
rmae_result_numpy = numpy_loss.rmae(y, y_hat, seasonality)
self.assertEqual(rmae_result_numpy, rmae_result_pytorch)
'''
def test_qloss(self):
ql_numpy = quantile_loss(self.y, self.y_hat, q=self.q)
ql_pytorch = QuantileLoss(self.y, self.y_hat, q=self.q).numpy()
self.assertAlmostEqual(ql_numpy, ql_pytorch, places=6)
def test_mqloss(self):
weights = np.ones_like(self.y)
# Check numpy mql with default weights
mql_w_numpy = mqloss(self.y, self.yq_hat,
quantiles=self.qs,
weights=weights)
mql_default_w_numpy = mqloss(self.y, self.yq_hat,
quantiles=self.qs)
self.assertAlmostEqual(mql_w_numpy, mql_default_w_numpy)
# Check pytorch MQLoss with default weights
mql_w_pytorch = MQLoss(self.y, self.yq_hat,
quantiles=self.qs,
mask=t.Tensor(weights)).numpy()
mql_default_w_pytorch = MQLoss(self.y, self.yq_hat,
quantiles=self.qs).numpy()
self.assertAlmostEqual(mql_w_pytorch, mql_default_w_pytorch)
# Check numpy/pytorch MQLoss weighted equality
weights[0,:] = 0
mql_w_numpy = mqloss(self.y, self.yq_hat,
quantiles=self.qs,
weights=weights)
mql_w_pytorch = MQLoss(self.y, self.yq_hat,
quantiles=self.qs,
mask=t.Tensor(weights)).numpy()
self.assertAlmostEqual(mql_w_numpy, mql_w_pytorch)
unittest.main(argv=[''], verbosity=2, exit=False)