In [1]:
import os
In [2]:
os.chdir("../core/")
In [3]:
%%capture
from load_datasets import *
from ensemble import *
from saliency import *
from scipy.stats import describe
In [4]:
(SVHN_trainX, SVHN_trainY),(SVHN_testX, SVHN_testY) = load_data("SVHN")
(MNIST_trainX, MNIST_trainY),(MNIST_testX, MNIST_testY) = load_data("MNIST")
NotMNIST = load_data("NotMNIST")
(CIFAR10_trainX, CIFAR10_trainY),(CIFAR10_testX, CIFAR10_testY) = load_data("CIFAR10")
WARNING:absl:Warning: Setting shuffle_files=True because split=TRAIN and shuffle_files=None. This behavior will be deprecated on 2019-08-06, at which point shuffle_files=False will be the default for all splits.
../../datasets/notMNIST_small/A
Could not read: ../../datasets/notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png : Could not find a format to read the specified file in single-image mode - it's ok, skipping.
../../datasets/notMNIST_small/B
../../datasets/notMNIST_small/C
../../datasets/notMNIST_small/D
../../datasets/notMNIST_small/E
../../datasets/notMNIST_small/F
Could not read: ../../datasets/notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png : Could not find a format to read the specified file in single-image mode - it's ok, skipping.
../../datasets/notMNIST_small/G
../../datasets/notMNIST_small/H
../../datasets/notMNIST_small/I
../../datasets/notMNIST_small/J
In [5]:
MNIST_trainX,MNIST_testX = pad_dataset(MNIST_trainX),pad_dataset(MNIST_testX)
NotMNIST = pad_dataset(NotMNIST)

Creating ensemble

In [6]:
CIFAR10_lenet = get_lenet(input_shape = np.shape(CIFAR10_trainX[0]))
train_network(network = CIFAR10_lenet, trainX = CIFAR10_trainX, trainY = CIFAR10_trainY)
CIFAR10_lenets = clone_network_into_ensemble(number_of_networks = 5, network = CIFAR10_lenet)
train_ensemble(ensemble = CIFAR10_lenets, trainX = CIFAR10_trainX, trainY = CIFAR10_trainY)
WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
Epoch 1/5
50000/50000 [==============================] - 13s 258us/step - loss: 1.8388 - acc: 0.3390
Epoch 2/5
50000/50000 [==============================] - 10s 198us/step - loss: 1.4689 - acc: 0.4751
Epoch 3/5
50000/50000 [==============================] - 10s 198us/step - loss: 1.3206 - acc: 0.5337
Epoch 4/5
50000/50000 [==============================] - 10s 196us/step - loss: 1.2093 - acc: 0.5724
Epoch 5/5
50000/50000 [==============================] - 10s 194us/step - loss: 1.1179 - acc: 0.6093
Epoch 1/5
50000/50000 [==============================] - 10s 210us/step - loss: 1.8604 - acc: 0.3304
Epoch 2/5
50000/50000 [==============================] - 10s 197us/step - loss: 1.5068 - acc: 0.4619
Epoch 3/5
50000/50000 [==============================] - 10s 196us/step - loss: 1.3512 - acc: 0.5201
Epoch 4/5
50000/50000 [==============================] - 10s 197us/step - loss: 1.2319 - acc: 0.5643
Epoch 5/5
50000/50000 [==============================] - 10s 196us/step - loss: 1.1354 - acc: 0.6013
Epoch 1/5
50000/50000 [==============================] - 11s 211us/step - loss: 1.8575 - acc: 0.3301
Epoch 2/5
50000/50000 [==============================] - 10s 202us/step - loss: 1.4598 - acc: 0.4807
Epoch 3/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.3018 - acc: 0.5366
Epoch 4/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.1878 - acc: 0.5815
Epoch 5/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.0892 - acc: 0.6181
Epoch 1/5
50000/50000 [==============================] - 10s 209us/step - loss: 1.8630 - acc: 0.3280
Epoch 2/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.4861 - acc: 0.4706
Epoch 3/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.3197 - acc: 0.5313
Epoch 4/5
50000/50000 [==============================] - 10s 200us/step - loss: 1.1995 - acc: 0.5782
Epoch 5/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.1005 - acc: 0.6142
Epoch 1/5
50000/50000 [==============================] - 11s 213us/step - loss: 1.8454 - acc: 0.3377
Epoch 2/5
50000/50000 [==============================] - 10s 205us/step - loss: 1.4875 - acc: 0.4690
Epoch 3/5
50000/50000 [==============================] - 10s 203us/step - loss: 1.3324 - acc: 0.5271
Epoch 4/5
50000/50000 [==============================] - 10s 203us/step - loss: 1.2207 - acc: 0.5701
Epoch 5/5
50000/50000 [==============================] - 10s 203us/step - loss: 1.1228 - acc: 0.6048
Epoch 1/5
50000/50000 [==============================] - 10s 210us/step - loss: 1.8347 - acc: 0.3412
Epoch 2/5
50000/50000 [==============================] - 10s 199us/step - loss: 1.4695 - acc: 0.4764
Epoch 3/5
50000/50000 [==============================] - 10s 201us/step - loss: 1.3184 - acc: 0.5337
Epoch 4/5
50000/50000 [==============================] - 10s 199us/step - loss: 1.2079 - acc: 0.5761
Epoch 5/5
50000/50000 [==============================] - 10s 200us/step - loss: 1.1122 - acc: 0.6088
Out[6]:
[<keras.engine.sequential.Sequential at 0x7f390005fed0>,
 <keras.engine.sequential.Sequential at 0x7f38c6201d50>,
 <keras.engine.sequential.Sequential at 0x7f38c60e7750>,
 <keras.engine.sequential.Sequential at 0x7f38c6027fd0>,
 <keras.engine.sequential.Sequential at 0x7f38c5f826d0>]
In [7]:
CIFAR10_optimizers = get_ensemble_optimisers(ensemble = CIFAR10_lenets)

Extracting data

CIFAR10

In [13]:
CIFAR10_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = CIFAR10_lenets, inputs = CIFAR10_testX, number_of_output_nodes = 10)
CIFAR10_predicted_outputs = get_ensemble_predicted_outputs(ensemble = CIFAR10_lenets, inputs = CIFAR10_testX, number_of_output_nodes = 10)
CIFAR10_output_values_of_predicted_outputs = np.take(CIFAR10_outputs_values,CIFAR10_predicted_outputs)
In [14]:
CIFAR10_ensemble_votes = get_ensemble_votes_for_multiple_inputs(CIFAR10_lenets,CIFAR10_testX,10)
CIFAR10_majority_voting_predicted_outputs = np.argmax(CIFAR10_ensemble_votes,axis = 1)
CIFAR10_vote_ratios_of_predicted_outputs = np.max(CIFAR10_ensemble_votes,axis = 1)/5
CIFAR10_different_instances_between_voting_and_averaging = np.nonzero(CIFAR10_majority_voting_predicted_outputs-CIFAR10_predicted_outputs)[0]
In [15]:
CIFAR10_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = CIFAR10_lenets,
    inputs = CIFAR10_testX,
    output_nodes = CIFAR10_predicted_outputs,
    optimisers = CIFAR10_optimizers,
    visualised_layer = 9)

CIFAR10_uncertainties = calculate_uncertainties_with_maps(CIFAR10_saliency_maps)
100%|██████████| 10000/10000 [00:38<00:00, 262.65it/s]
100%|██████████| 10000/10000 [00:37<00:00, 264.03it/s]
100%|██████████| 10000/10000 [00:38<00:00, 262.84it/s]
100%|██████████| 10000/10000 [00:37<00:00, 265.40it/s]
100%|██████████| 10000/10000 [00:37<00:00, 266.18it/s]
In [16]:
CIFAR10_singular_predictions_values = CIFAR10_lenet.predict(CIFAR10_testX)
In [17]:
CIFAR10_variance = np.var(CIFAR10_singular_predictions_values,axis = 1)

SVHN

In [8]:
SVHN_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = CIFAR10_lenets, inputs = SVHN_testX, number_of_output_nodes = 10)
SVHN_predicted_outputs = get_ensemble_predicted_outputs(ensemble = CIFAR10_lenets, inputs = SVHN_testX, number_of_output_nodes = 10)
SVHN_output_values_of_predicted_outputs = np.take(SVHN_outputs_values,SVHN_predicted_outputs)
In [9]:
SVHN_ensemble_votes = get_ensemble_votes_for_multiple_inputs(CIFAR10_lenets,SVHN_testX,10)
SVHN_majority_voting_predicted_outputs = np.argmax(SVHN_ensemble_votes,axis = 1)
SVHN_vote_ratios_of_predicted_outputs = np.max(SVHN_ensemble_votes,axis = 1)/5
SVHN_different_instances_between_voting_and_averaging = np.nonzero(SVHN_majority_voting_predicted_outputs-SVHN_predicted_outputs)[0]
In [10]:
SVHN_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = CIFAR10_lenets,
    inputs = SVHN_testX,
    output_nodes = SVHN_predicted_outputs,
    optimisers = CIFAR10_optimizers,
    visualised_layer = 9)

SVHN_uncertainties = calculate_uncertainties_with_maps(SVHN_saliency_maps)
100%|██████████| 26032/26032 [01:38<00:00, 263.53it/s]
100%|██████████| 26032/26032 [01:38<00:00, 264.16it/s]
100%|██████████| 26032/26032 [01:37<00:00, 267.40it/s]
100%|██████████| 26032/26032 [01:37<00:00, 267.17it/s]
100%|██████████| 26032/26032 [01:37<00:00, 266.98it/s]
In [11]:
SVHN_singular_predictions_values = CIFAR10_lenet.predict(SVHN_testX)
In [12]:
SVHN_variance = np.var(SVHN_singular_predictions_values,axis = 1)

MNIST

In [18]:
MNIST_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = CIFAR10_lenets, inputs = MNIST_testX, number_of_output_nodes = 10)
MNIST_predicted_outputs = get_ensemble_predicted_outputs(ensemble = CIFAR10_lenets, inputs = MNIST_testX, number_of_output_nodes = 10)
MNIST_output_values_of_predicted_outputs = np.take(MNIST_outputs_values,MNIST_predicted_outputs)
In [19]:
MNIST_ensemble_votes = get_ensemble_votes_for_multiple_inputs(CIFAR10_lenets,MNIST_testX,10)
MNIST_majority_voting_predicted_outputs = np.argmax(MNIST_ensemble_votes,axis = 1)
MNIST_vote_ratios_of_predicted_outputs = np.max(MNIST_ensemble_votes,axis = 1)/5
MNIST_different_instances_between_voting_and_averaging = np.nonzero(MNIST_majority_voting_predicted_outputs-MNIST_predicted_outputs)[0]
In [20]:
MNIST_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = CIFAR10_lenets,
    inputs = MNIST_testX,
    output_nodes = MNIST_predicted_outputs,
    optimisers = CIFAR10_optimizers,
    visualised_layer = 9)

MNIST_uncertainties = calculate_uncertainties_with_maps(MNIST_saliency_maps)
100%|██████████| 10000/10000 [00:37<00:00, 268.80it/s]
100%|██████████| 10000/10000 [00:37<00:00, 269.59it/s]
100%|██████████| 10000/10000 [00:37<00:00, 269.25it/s]
100%|██████████| 10000/10000 [00:37<00:00, 268.73it/s]
100%|██████████| 10000/10000 [00:37<00:00, 267.19it/s]
In [21]:
MNIST_singular_predictions_values = CIFAR10_lenet.predict(MNIST_testX)
In [22]:
MNIST_variance = np.var(MNIST_singular_predictions_values,axis = 1)

NotMNIST

In [23]:
NotMNIST_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = CIFAR10_lenets, inputs = NotMNIST, number_of_output_nodes = 10)
NotMNIST_predicted_outputs = get_ensemble_predicted_outputs(ensemble = CIFAR10_lenets, inputs = NotMNIST, number_of_output_nodes = 10)
NotMNIST_output_values_of_predicted_outputs = np.take(NotMNIST_outputs_values,NotMNIST_predicted_outputs)
In [24]:
NotMNIST_ensemble_votes = get_ensemble_votes_for_multiple_inputs(CIFAR10_lenets,NotMNIST,10)
NotMNIST_majority_voting_predicted_outputs = np.argmax(NotMNIST_ensemble_votes,axis = 1)
NotMNIST_vote_ratios_of_predicted_outputs = np.max(NotMNIST_ensemble_votes,axis = 1)/5
NotMNIST_different_instances_between_voting_and_averaging = np.nonzero(NotMNIST_majority_voting_predicted_outputs-NotMNIST_predicted_outputs)[0]
In [25]:
NotMNIST_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = CIFAR10_lenets,
    inputs = NotMNIST,
    output_nodes = NotMNIST_predicted_outputs,
    optimisers = CIFAR10_optimizers,
    visualised_layer = 9)

NotMNIST_uncertainties = calculate_uncertainties_with_maps(NotMNIST_saliency_maps)
100%|██████████| 18724/18724 [01:10<00:00, 267.43it/s]
100%|██████████| 18724/18724 [01:09<00:00, 269.05it/s]
100%|██████████| 18724/18724 [01:09<00:00, 269.49it/s]
100%|██████████| 18724/18724 [01:09<00:00, 269.34it/s]
100%|██████████| 18724/18724 [01:09<00:00, 268.00it/s]
In [26]:
NotMNIST_singular_predictions_values = CIFAR10_lenet.predict(NotMNIST)
In [27]:
NotMNIST_variance = np.var(NotMNIST_singular_predictions_values,axis = 1)

Exploring data

In [28]:
plt.style.use('default')
In [30]:
# General evaluation
print("Evaluation (loss,acc) of singular LeNet:")
print(CIFAR10_lenet.evaluate(CIFAR10_testX, CIFAR10_testY))
print("")
print("Evaluation (loss,acc) of each LeNet in the ensemble:")
for lenet in CIFAR10_lenets:
    print(lenet.evaluate(CIFAR10_testX, CIFAR10_testY))
print("")
print("Accuracy of ensemble:")
print(evaluate_ensemble_accuracy(CIFAR10_lenets,CIFAR10_testX,CIFAR10_testY))
print("")
print("Averaged softmax output values of any output node of the ensemble:")
print(describe(CIFAR10_outputs_values,axis=None))
print("")
print("Averaged softmax output values of the ensemble's predicted outputs:")
print(describe(CIFAR10_output_values_of_predicted_outputs))
print("")
print("Softmax output values of the singular LeNet's outputs:")
print(describe(CIFAR10_lenet.predict(CIFAR10_testX),axis = None))
print("")
Evaluation (loss,acc) of singular LeNet:
10000/10000 [==============================] - 1s 134us/step
[1.2714656345367432, 0.5603]

Evaluation (loss,acc) of each LeNet in the ensemble:
10000/10000 [==============================] - 1s 136us/step
[1.1562888818740844, 0.5911]
10000/10000 [==============================] - 1s 135us/step
[1.1129447187423707, 0.615]
10000/10000 [==============================] - 1s 133us/step
[1.1596617881774902, 0.5867]
10000/10000 [==============================] - 1s 134us/step
[1.147736731338501, 0.5928]
10000/10000 [==============================] - 1s 133us/step
[1.170890924835205, 0.5786]

Accuracy of ensemble:
0.6329

Averaged softmax output values of any output node of the ensemble:
DescribeResult(nobs=100000, minmax=(8.768325038266411e-09, 0.9997996330261231), mean=0.10000000001693274, variance=0.033957832444253884, skewness=2.675616743064035, kurtosis=7.14828613408079)

Averaged softmax output values of the ensemble's predicted outputs:
DescribeResult(nobs=10000, minmax=(0.001072170922998339, 0.5114781379699707), mean=0.08990553649252046, variance=0.02323553574131141, skewness=1.9895335246995545, kurtosis=2.492794172939414)

Softmax output values of the singular LeNet's outputs:
DescribeResult(nobs=100000, minmax=(5.6647664e-10, 0.99988437), mean=0.10000001, variance=0.03539974, skewness=2.7398009300231934, kurtosis=7.488148010705743)

CIFAR10

In [31]:
print("Statistical description of uncertainty values:")
print(describe(CIFAR10_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(CIFAR10_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=10000, minmax=(0.34788661432688517, 0.5052721632436987), mean=0.4106217056221818, variance=0.0003668963384683274, skewness=0.32762306007533337, kurtosis=0.3845758465589304)
Histogram of uncertainty values:
In [32]:
plt.scatter(x = CIFAR10_output_values_of_predicted_outputs, y = CIFAR10_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [33]:
print("Variance of singular LeNet's outputs for each input")
print(describe(CIFAR10_variance))
plt.hist(CIFAR10_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=10000, minmax=(0.0012816834, 0.08997686), mean=0.035399385, variance=0.00054012286, skewness=0.719569206237793, kurtosis=-0.5591370876462562)
In [34]:
print(len(CIFAR10_different_instances_between_voting_and_averaging))
print(1 - len(CIFAR10_different_instances_between_voting_and_averaging)/len(CIFAR10_testX))
print(describe(CIFAR10_vote_ratios_of_predicted_outputs))
plt.hist(CIFAR10_vote_ratios_of_predicted_outputs)
plt.show()
797
0.9203
DescribeResult(nobs=10000, minmax=(0.2, 1.0), mean=0.8124600000000001, variance=0.04125287368736874, skewness=-0.6513388190608719, kurtosis=-0.8207471744377908)

SVHN

In [35]:
print("Statistical description of uncertainty values:")
print(describe(SVHN_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(SVHN_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=26032, minmax=(0.3549333184677796, 0.5127161169375014), mean=0.4174281233509731, variance=0.0003261761185069903, skewness=0.319797925270507, kurtosis=0.48922143470835966)
Histogram of uncertainty values:
In [36]:
plt.scatter(x = SVHN_output_values_of_predicted_outputs, y = SVHN_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [37]:
print("Variance of singular LeNet's outputs for each input")
print(describe(SVHN_variance))
plt.hist(SVHN_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=26032, minmax=(0.0011701649, 0.08623314), mean=0.016745832, variance=0.00011809687, skewness=1.8079102039337158, kurtosis=4.117979119577037)
In [38]:
print(len(SVHN_different_instances_between_voting_and_averaging))
print(1 - len(SVHN_different_instances_between_voting_and_averaging)/len(SVHN_testX))
print(describe(SVHN_vote_ratios_of_predicted_outputs))
plt.hist(SVHN_vote_ratios_of_predicted_outputs)
plt.show()
3107
0.8806468961278426
DescribeResult(nobs=26032, minmax=(0.2, 1.0), mean=0.7243546404425323, variance=0.04090811562980425, skewness=-0.072135818637783, kurtosis=-1.07412389676847)

MNIST

In [39]:
print("Statistical description of uncertainty values:")
print(describe(MNIST_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(MNIST_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=10000, minmax=(0.38286950856798035, 0.5422022497285794), mean=0.4366005203921488, variance=0.00041326049772408697, skewness=0.6662529081211062, kurtosis=0.7371768678244672)
Histogram of uncertainty values:
In [40]:
plt.scatter(x = MNIST_output_values_of_predicted_outputs, y = MNIST_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [41]:
print("Variance of singular LeNet's outputs for each input")
print(describe(MNIST_variance))
plt.hist(MNIST_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=10000, minmax=(0.0030988047, 0.08748043), mean=0.029887345, variance=0.000284869, skewness=0.9558697938919067, kurtosis=0.3232028566218532)
In [42]:
print(len(MNIST_different_instances_between_voting_and_averaging))
print(1 - len(MNIST_different_instances_between_voting_and_averaging)/len(MNIST_testX))
print(describe(MNIST_vote_ratios_of_predicted_outputs))
plt.hist(MNIST_vote_ratios_of_predicted_outputs)
plt.show()
1285
0.8714999999999999
DescribeResult(nobs=10000, minmax=(0.2, 1.0), mean=0.6998800000000001, variance=0.046516637263726374, skewness=0.019254604977825995, kurtosis=-1.1648177494331233)

NotMNIST

In [43]:
print("Statistical description of uncertainty values:")
print(describe(NotMNIST_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(NotMNIST_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=18724, minmax=(0.36439394386512447, 0.5357528751727149), mean=0.43776521835542526, variance=0.00034324129995451206, skewness=0.3770125667857473, kurtosis=0.5480843378077167)
Histogram of uncertainty values:
In [44]:
plt.scatter(x = NotMNIST_output_values_of_predicted_outputs, y = NotMNIST_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [45]:
print("Variance of singular LeNet's outputs for each input")
print(describe(NotMNIST_variance))
plt.hist(NotMNIST_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=18724, minmax=(0.0031909193, 0.08993502), mean=0.039079987, variance=0.00044720276, skewness=0.6503708958625793, kurtosis=-0.5701119235508836)
In [46]:
print(describe(NotMNIST_vote_ratios_of_predicted_outputs))
plt.hist(NotMNIST_vote_ratios_of_predicted_outputs)
plt.show()
DescribeResult(nobs=18724, minmax=(0.2, 1.0), mean=0.7366374706259347, variance=0.04257579955939801, skewness=-0.19015989470245415, kurtosis=-1.0902500428224708)