In [1]:
import os
In [2]:
os.chdir("../core/")
In [3]:
%%capture
from load_datasets import *
from ensemble import *
from saliency import *
from scipy.stats import describe
In [4]:
(SVHN_trainX, SVHN_trainY),(SVHN_testX, SVHN_testY) = load_data("SVHN")
(MNIST_trainX, MNIST_trainY),(MNIST_testX, MNIST_testY) = load_data("MNIST")
NotMNIST = load_data("NotMNIST")
(CIFAR10_trainX, CIFAR10_trainY),(CIFAR10_testX, CIFAR10_testY) = load_data("CIFAR10")
WARNING:absl:Warning: Setting shuffle_files=True because split=TRAIN and shuffle_files=None. This behavior will be deprecated on 2019-08-06, at which point shuffle_files=False will be the default for all splits.
../../datasets/notMNIST_small/A
Could not read: ../../datasets/notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png : Could not find a format to read the specified file in single-image mode - it's ok, skipping.
../../datasets/notMNIST_small/B
../../datasets/notMNIST_small/C
../../datasets/notMNIST_small/D
../../datasets/notMNIST_small/E
../../datasets/notMNIST_small/F
Could not read: ../../datasets/notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png : Could not find a format to read the specified file in single-image mode - it's ok, skipping.
../../datasets/notMNIST_small/G
../../datasets/notMNIST_small/H
../../datasets/notMNIST_small/I
../../datasets/notMNIST_small/J
In [5]:
SVHN_trainX,SVHN_testX,CIFAR10_trainX,CIFAR10_testX = crop_32_to_28(SVHN_trainX),crop_32_to_28(SVHN_testX),crop_32_to_28(CIFAR10_trainX),crop_32_to_28(CIFAR10_testX)
SVHN_trainX,SVHN_testX,CIFAR10_trainX,CIFAR10_testX = make_monochrome(SVHN_trainX),make_monochrome(SVHN_testX),make_monochrome(CIFAR10_trainX),make_monochrome(CIFAR10_testX)

Creating ensemble

In [6]:
MNIST_lenet = get_lenet(input_shape = np.shape(MNIST_trainX[0]))
train_network(network = MNIST_lenet, trainX = MNIST_trainX, trainY = MNIST_trainY)
MNIST_lenets = clone_network_into_ensemble(number_of_networks = 5, network = MNIST_lenet)
train_ensemble(ensemble = MNIST_lenets, trainX = MNIST_trainX, trainY = MNIST_trainY)
WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.

WARNING:tensorflow:From /users/ai/u7ysrv/anaconda3/envs/samel/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
Epoch 1/5
60000/60000 [==============================] - 15s 247us/step - loss: 0.4133 - acc: 0.8835
Epoch 2/5
60000/60000 [==============================] - 11s 191us/step - loss: 0.1077 - acc: 0.9676
Epoch 3/5
60000/60000 [==============================] - 12s 193us/step - loss: 0.0743 - acc: 0.9768
Epoch 4/5
60000/60000 [==============================] - 11s 189us/step - loss: 0.0586 - acc: 0.9822
Epoch 5/5
60000/60000 [==============================] - 11s 188us/step - loss: 0.0486 - acc: 0.9852
Epoch 1/5
60000/60000 [==============================] - 12s 202us/step - loss: 0.4022 - acc: 0.8831
Epoch 2/5
60000/60000 [==============================] - 12s 195us/step - loss: 0.1061 - acc: 0.9682
Epoch 3/5
60000/60000 [==============================] - 12s 192us/step - loss: 0.0734 - acc: 0.9782
Epoch 4/5
60000/60000 [==============================] - 12s 193us/step - loss: 0.0580 - acc: 0.9827
Epoch 5/5
60000/60000 [==============================] - 11s 190us/step - loss: 0.0489 - acc: 0.9851
Epoch 1/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.4144 - acc: 0.8786
Epoch 2/5
60000/60000 [==============================] - 12s 195us/step - loss: 0.1121 - acc: 0.9662
Epoch 3/5
60000/60000 [==============================] - 12s 193us/step - loss: 0.0780 - acc: 0.9761
Epoch 4/5
60000/60000 [==============================] - 12s 194us/step - loss: 0.0615 - acc: 0.9812
Epoch 5/5
60000/60000 [==============================] - 12s 194us/step - loss: 0.0512 - acc: 0.9841
Epoch 1/5
60000/60000 [==============================] - 12s 206us/step - loss: 0.3988 - acc: 0.8842
Epoch 2/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.1101 - acc: 0.9672
Epoch 3/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.0750 - acc: 0.9772
Epoch 4/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.0597 - acc: 0.9816
Epoch 5/5
60000/60000 [==============================] - 12s 199us/step - loss: 0.0502 - acc: 0.9848
Epoch 1/5
60000/60000 [==============================] - 12s 204us/step - loss: 0.4234 - acc: 0.8804
Epoch 2/5
60000/60000 [==============================] - 12s 195us/step - loss: 0.1052 - acc: 0.9683
Epoch 3/5
60000/60000 [==============================] - 12s 193us/step - loss: 0.0741 - acc: 0.9772
Epoch 4/5
60000/60000 [==============================] - 12s 197us/step - loss: 0.0601 - acc: 0.9812
Epoch 5/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.0499 - acc: 0.9847
Epoch 1/5
60000/60000 [==============================] - 12s 207us/step - loss: 0.4284 - acc: 0.8788
Epoch 2/5
60000/60000 [==============================] - 12s 199us/step - loss: 0.1074 - acc: 0.9677
Epoch 3/5
60000/60000 [==============================] - 12s 197us/step - loss: 0.0754 - acc: 0.9769
Epoch 4/5
60000/60000 [==============================] - 12s 198us/step - loss: 0.0600 - acc: 0.9820
Epoch 5/5
60000/60000 [==============================] - 12s 199us/step - loss: 0.0503 - acc: 0.9847
Out[6]:
[<keras.engine.sequential.Sequential at 0x7f015029d390>,
 <keras.engine.sequential.Sequential at 0x7f01a81e9910>,
 <keras.engine.sequential.Sequential at 0x7f01a80c1410>,
 <keras.engine.sequential.Sequential at 0x7f01a8005d50>,
 <keras.engine.sequential.Sequential at 0x7f01a0759410>]
In [7]:
MNIST_optimizers = get_ensemble_optimisers(ensemble = MNIST_lenets)

Extracting data

MNIST

In [8]:
MNIST_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = MNIST_lenets, inputs = MNIST_testX, number_of_output_nodes = 10)
MNIST_predicted_outputs = get_ensemble_predicted_outputs(ensemble = MNIST_lenets, inputs = MNIST_testX, number_of_output_nodes = 10)
MNIST_output_values_of_predicted_outputs = np.take(MNIST_outputs_values,MNIST_predicted_outputs)
In [9]:
MNIST_ensemble_votes = get_ensemble_votes_for_multiple_inputs(MNIST_lenets,MNIST_testX,10)
MNIST_majority_voting_predicted_outputs = np.argmax(MNIST_ensemble_votes,axis = 1)
MNIST_vote_ratios_of_predicted_outputs = np.max(MNIST_ensemble_votes,axis = 1)/5
MNIST_different_instances_between_voting_and_averaging = np.nonzero(MNIST_majority_voting_predicted_outputs-MNIST_predicted_outputs)[0]
In [10]:
MNIST_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = MNIST_lenets,
    inputs = MNIST_testX,
    output_nodes = MNIST_predicted_outputs,
    optimisers = MNIST_optimizers,
    visualised_layer = 9)

MNIST_uncertainties = calculate_uncertainties_with_maps(MNIST_saliency_maps)
100%|██████████| 10000/10000 [00:36<00:00, 275.05it/s]
100%|██████████| 10000/10000 [00:36<00:00, 273.99it/s]
100%|██████████| 10000/10000 [00:36<00:00, 276.61it/s]
100%|██████████| 10000/10000 [00:36<00:00, 274.01it/s]
100%|██████████| 10000/10000 [00:36<00:00, 273.27it/s]
In [11]:
MNIST_singular_predictions_values = MNIST_lenet.predict(MNIST_testX)
In [12]:
MNIST_variance = np.var(MNIST_singular_predictions_values,axis = 1)

NotMNIST

In [13]:
NotMNIST_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = MNIST_lenets, inputs = NotMNIST, number_of_output_nodes = 10)
NotMNIST_predicted_outputs = get_ensemble_predicted_outputs(ensemble = MNIST_lenets, inputs = NotMNIST, number_of_output_nodes = 10)
NotMNIST_output_values_of_predicted_outputs = np.take(NotMNIST_outputs_values,NotMNIST_predicted_outputs)
In [14]:
NotMNIST_ensemble_votes = get_ensemble_votes_for_multiple_inputs(MNIST_lenets,NotMNIST,10)
NotMNIST_majority_voting_predicted_outputs = np.argmax(NotMNIST_ensemble_votes,axis = 1)
NotMNIST_vote_ratios_of_predicted_outputs = np.max(NotMNIST_ensemble_votes,axis = 1)/5
NotMNIST_different_instances_between_voting_and_averaging = np.nonzero(NotMNIST_majority_voting_predicted_outputs-NotMNIST_predicted_outputs)[0]
In [15]:
NotMNIST_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = MNIST_lenets,
    inputs = NotMNIST,
    output_nodes = NotMNIST_predicted_outputs,
    optimisers = MNIST_optimizers,
    visualised_layer = 9)

NotMNIST_uncertainties = calculate_uncertainties_with_maps(NotMNIST_saliency_maps)
100%|██████████| 18724/18724 [01:05<00:00, 285.11it/s]
100%|██████████| 18724/18724 [01:05<00:00, 285.74it/s]
100%|██████████| 18724/18724 [01:05<00:00, 285.98it/s]
100%|██████████| 18724/18724 [01:05<00:00, 285.24it/s]
100%|██████████| 18724/18724 [01:05<00:00, 285.33it/s]
In [16]:
NotMNIST_singular_predictions_values = MNIST_lenet.predict(NotMNIST)
In [17]:
NotMNIST_variance = np.var(NotMNIST_singular_predictions_values,axis = 1)

SVHN

In [18]:
SVHN_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = MNIST_lenets, inputs = SVHN_testX, number_of_output_nodes = 10)
SVHN_predicted_outputs = get_ensemble_predicted_outputs(ensemble = MNIST_lenets, inputs = SVHN_testX, number_of_output_nodes = 10)
SVHN_output_values_of_predicted_outputs = np.take(SVHN_outputs_values,SVHN_predicted_outputs)
In [19]:
SVHN_ensemble_votes = get_ensemble_votes_for_multiple_inputs(MNIST_lenets,SVHN_testX,10)
SVHN_majority_voting_predicted_outputs = np.argmax(SVHN_ensemble_votes,axis = 1)
SVHN_vote_ratios_of_predicted_outputs = np.max(SVHN_ensemble_votes,axis = 1)/5
SVHN_different_instances_between_voting_and_averaging = np.nonzero(SVHN_majority_voting_predicted_outputs-SVHN_predicted_outputs)[0]
In [20]:
SVHN_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = MNIST_lenets,
    inputs = SVHN_testX,
    output_nodes = SVHN_predicted_outputs,
    optimisers = MNIST_optimizers,
    visualised_layer = 9)

SVHN_uncertainties = calculate_uncertainties_with_maps(SVHN_saliency_maps)
100%|██████████| 26032/26032 [01:31<00:00, 284.79it/s]
100%|██████████| 26032/26032 [01:31<00:00, 285.78it/s]
100%|██████████| 26032/26032 [01:31<00:00, 285.96it/s]
100%|██████████| 26032/26032 [01:30<00:00, 287.34it/s]
100%|██████████| 26032/26032 [01:31<00:00, 285.15it/s]
In [21]:
SVHN_singular_predictions_values = MNIST_lenet.predict(SVHN_testX)
In [22]:
SVHN_variance = np.var(SVHN_singular_predictions_values,axis = 1)

CIFAR10

In [23]:
CIFAR10_outputs_values = get_ensemble_output_values_for_multiple_inputs(ensemble = MNIST_lenets, inputs = CIFAR10_testX, number_of_output_nodes = 10)
CIFAR10_predicted_outputs = get_ensemble_predicted_outputs(ensemble = MNIST_lenets, inputs = CIFAR10_testX, number_of_output_nodes = 10)
CIFAR10_output_values_of_predicted_outputs = np.take(CIFAR10_outputs_values,CIFAR10_predicted_outputs)
In [24]:
CIFAR10_ensemble_votes = get_ensemble_votes_for_multiple_inputs(MNIST_lenets,CIFAR10_testX,10)
CIFAR10_majority_voting_predicted_outputs = np.argmax(CIFAR10_ensemble_votes,axis = 1)
CIFAR10_vote_ratios_of_predicted_outputs = np.max(CIFAR10_ensemble_votes,axis = 1)/5
CIFAR10_different_instances_between_voting_and_averaging = np.nonzero(CIFAR10_majority_voting_predicted_outputs-CIFAR10_predicted_outputs)[0]
In [25]:
CIFAR10_saliency_maps = generate_ensemble_saliency_maps_for_multiple_inputs(
    ensemble = MNIST_lenets,
    inputs = CIFAR10_testX,
    output_nodes = CIFAR10_predicted_outputs,
    optimisers = MNIST_optimizers,
    visualised_layer = 9)

CIFAR10_uncertainties = calculate_uncertainties_with_maps(CIFAR10_saliency_maps)
100%|██████████| 10000/10000 [00:34<00:00, 287.67it/s]
100%|██████████| 10000/10000 [00:34<00:00, 286.62it/s]
100%|██████████| 10000/10000 [00:35<00:00, 284.57it/s]
100%|██████████| 10000/10000 [00:35<00:00, 284.75it/s]
100%|██████████| 10000/10000 [00:34<00:00, 288.59it/s]
In [26]:
CIFAR10_singular_predictions_values = MNIST_lenet.predict(CIFAR10_testX)
In [27]:
CIFAR10_variance = np.var(CIFAR10_singular_predictions_values,axis = 1)

Exploring data

In [28]:
# General evaluation
print("Evaluation (loss,acc) of singular LeNet:")
print(MNIST_lenet.evaluate(MNIST_testX, MNIST_testY))
print("")
print("Evaluation (loss,acc) of each LeNet in the ensemble:")
for lenet in MNIST_lenets:
    print(lenet.evaluate(MNIST_testX, MNIST_testY))
print("")
print("Accuracy of ensemble:")
print(evaluate_ensemble_accuracy(MNIST_lenets,MNIST_testX,MNIST_testY))
print("")
print("Averaged softmax output values of any output node of the ensemble:")
print(describe(MNIST_outputs_values,axis=None))
print("")
print("Averaged softmax output values of the ensemble's predicted outputs:")
print(describe(MNIST_output_values_of_predicted_outputs))
print("")
print("Softmax output values of the singular LeNet's outputs:")
print(describe(MNIST_lenet.predict(MNIST_testX),axis = None))
print("")
Evaluation (loss,acc) of singular LeNet:
10000/10000 [==============================] - 1s 118us/step
[0.03957363048342522, 0.9865]

Evaluation (loss,acc) of each LeNet in the ensemble:
10000/10000 [==============================] - 1s 119us/step
[0.0404068144927267, 0.9865]
10000/10000 [==============================] - 1s 115us/step
[0.04917081510843709, 0.9829]
10000/10000 [==============================] - 1s 115us/step
[0.03850150211374275, 0.9874]
10000/10000 [==============================] - 1s 116us/step
[0.04425659646950662, 0.9849]
10000/10000 [==============================] - 1s 115us/step
[0.046176857580710204, 0.9842]

Accuracy of ensemble:
0.9882

Averaged softmax output values of any output node of the ensemble:
DescribeResult(nobs=100000, minmax=(5.675064865383555e-18, 1.0), mean=0.10000000027772643, variance=0.08697004136574216, skewness=2.671839476307086, kurtosis=5.187432493178379)

Averaged softmax output values of the ensemble's predicted outputs:
DescribeResult(nobs=10000, minmax=(2.741314120147109e-13, 0.9999272942543029), mean=0.10259992685012191, variance=0.09206754339120184, skewness=2.6193355862266734, kurtosis=4.860918941866753)

Softmax output values of the singular LeNet's outputs:
DescribeResult(nobs=100000, minmax=(2.6719518e-18, 1.0), mean=0.1, variance=0.08751218, skewness=2.67099666595459, kurtosis=5.17309492763944)

MNIST

In [29]:
plt.style.use('default')
In [30]:
print("Statistical description of uncertainty values:")
print(describe(MNIST_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(MNIST_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=10000, minmax=(0.49518765383766966, 0.6754739046871886), mean=0.5720307108916417, variance=0.0005060723805718031, skewness=0.29348661989592956, kurtosis=-0.056711631476950686)
Histogram of uncertainty values:
In [31]:
plt.scatter(x = MNIST_output_values_of_predicted_outputs, y = MNIST_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [32]:
print("Variance of singular LeNet's outputs for each input")
print(describe(MNIST_variance))
plt.hist(MNIST_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=10000, minmax=(0.011370154, 0.08999999), mean=0.0875113, variance=7.506578e-05, skewness=-4.882788181304932, kurtosis=25.631258138131443)
In [33]:
print(len(MNIST_different_instances_between_voting_and_averaging))
print(1 - len(MNIST_different_instances_between_voting_and_averaging)/len(MNIST_testX))
print(describe(MNIST_vote_ratios_of_predicted_outputs))
plt.hist(MNIST_vote_ratios_of_predicted_outputs)
plt.show()
23
0.9977
DescribeResult(nobs=10000, minmax=(0.4, 1.0), mean=0.9924, variance=0.002542494249424942, skewness=-7.38327373620055, kurtosis=58.41205269639184)

NotMNIST

In [34]:
print("Statistical description of uncertainty values:")
print(describe(NotMNIST_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(NotMNIST_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=18724, minmax=(0.5171966305315824, 0.8634043473348401), mean=0.6231890778698185, variance=0.000876007096892642, skewness=0.4238667416862439, kurtosis=1.521598411770011)
Histogram of uncertainty values:
In [35]:
plt.scatter(x = NotMNIST_output_values_of_predicted_outputs, y = NotMNIST_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [36]:
print("Variance of singular LeNet's outputs for each input")
print(describe(NotMNIST_variance))
plt.hist(NotMNIST_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=18724, minmax=(7.545685e-05, 0.08999447), mean=0.041062996, variance=0.0006358567, skewness=0.3895334005355835, kurtosis=-0.9779303212338646)
In [37]:
print(describe(NotMNIST_vote_ratios_of_predicted_outputs))
plt.hist(NotMNIST_vote_ratios_of_predicted_outputs)
plt.show()
DescribeResult(nobs=18724, minmax=(0.2, 1.0), mean=0.8063982055116428, variance=0.039326683489809604, skewness=-0.5742058746663986, kurtosis=-0.854284359084005)

SVHN

In [38]:
print("Statistical description of uncertainty values:")
print(describe(SVHN_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(SVHN_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=26032, minmax=(0.493116849104918, 0.7040947180216756), mean=0.6063421945688233, variance=0.0006680564612665946, skewness=-0.821603496729206, kurtosis=1.3403920856828702)
Histogram of uncertainty values:
In [39]:
plt.scatter(x = SVHN_output_values_of_predicted_outputs, y = SVHN_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [40]:
print("Variance of singular LeNet's outputs for each input")
print(describe(SVHN_variance))
plt.hist(SVHN_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=26032, minmax=(0.0011654603, 0.0899778), mean=0.021770248, variance=0.00028466663, skewness=1.6435461044311523, kurtosis=2.5933373804977)
In [41]:
print(len(SVHN_different_instances_between_voting_and_averaging))
print(1 - len(SVHN_different_instances_between_voting_and_averaging)/len(SVHN_testX))
print(describe(SVHN_vote_ratios_of_predicted_outputs))
plt.hist(SVHN_vote_ratios_of_predicted_outputs)
plt.show()
1246
0.9521358328211432
DescribeResult(nobs=26032, minmax=(0.2, 1.0), mean=0.8726720958819915, variance=0.03328027847606502, skewness=-1.1740345407023591, kurtosis=0.17386446246882858)

CIFAR10

In [42]:
print("Statistical description of uncertainty values:")
print(describe(CIFAR10_uncertainties))
print("Histogram of uncertainty values:")
plt.hist(CIFAR10_uncertainties)
plt.show()
Statistical description of uncertainty values:
DescribeResult(nobs=10000, minmax=(0.5074232261268208, 0.6932457084043036), mean=0.5947107842345808, variance=0.0006956356571944476, skewness=-0.17508532978134472, kurtosis=-0.24432879145917052)
Histogram of uncertainty values:
In [43]:
plt.scatter(x = CIFAR10_output_values_of_predicted_outputs, y = CIFAR10_uncertainties)
print("Scatterplot of softmax value of predicted output (x) and uncertainty (y):")
plt.show()
Scatterplot of softmax value of predicted output (x) and uncertainty (y):
In [44]:
print("Variance of singular LeNet's outputs for each input")
print(describe(CIFAR10_variance))
plt.hist(CIFAR10_variance)
plt.show()
Variance of singular LeNet's outputs for each input
DescribeResult(nobs=10000, minmax=(0.0029217082, 0.08998102), mean=0.031015832, variance=0.00042199058, skewness=1.027807593345642, kurtosis=0.2550168613341768)
In [45]:
print(len(CIFAR10_different_instances_between_voting_and_averaging))
print(1 - len(CIFAR10_different_instances_between_voting_and_averaging)/len(CIFAR10_testX))
print(describe(CIFAR10_vote_ratios_of_predicted_outputs))
plt.hist(CIFAR10_vote_ratios_of_predicted_outputs)
plt.show()
775
0.9225
DescribeResult(nobs=10000, minmax=(0.2, 1.0), mean=0.80562, variance=0.04147256285628562, skewness=-0.545030662122984, kurtosis=-0.9764446502528545)