PerfKitBenchmarker Comparison

f	1	--cloud=GCP	f	1	--cloud=GCP
	2	--machine_type=n1-standard-4		2	--machine_type=n1-standard-4
	3	--benchmarks=netperf_simple		3	--benchmarks=netperf_simple
t	4	--json_path=/tmp/tmpxwVNUP.json	t	4	--json_path=/tmp/tmpk_UneO.json

Result comparison

	Base: json-path (v0.7.1-1-g725a7d7)				Head: netperf-csv (v0.7.1-4-g36b0d85)				Difference	Difference (relative)
1	TCP_RR_Transaction_Rate	netperf_simple	1280.87	transactions_per_second	TCP_RR_Transaction_Rate	netperf_simple	1333.62	transactions_per_second	52.75	1.041x
2	TCP_RR_Transaction_Rate	netperf_simple	4030.92	transactions_per_second	TCP_RR_Transaction_Rate	netperf_simple	4087.38	transactions_per_second	56.46	1.014x
3	TCP_CRR_Transaction_Rate	netperf_simple	373.6	transactions_per_second	TCP_CRR_Transaction_Rate	netperf_simple	404.17	transactions_per_second	30.57	1.082x
4	TCP_CRR_Transaction_Rate	netperf_simple	1270.88	transactions_per_second	TCP_CRR_Transaction_Rate	netperf_simple	1251.75	transactions_per_second	-19.13	0.985x
5	TCP_STREAM_Throughput	netperf_simple	1047.18	Mbits/sec	TCP_STREAM_Throughput	netperf_simple	1192.43	Mbits/sec	145.25	1.139x
6	TCP_STREAM_Throughput	netperf_simple	5095.9	Mbits/sec	TCP_STREAM_Throughput	netperf_simple	2804.9	Mbits/sec	-2291.0	0.550x
7	UDP_RR_Transaction_Rate	netperf_simple	1450.57	transactions_per_second	UDP_RR_Transaction_Rate	netperf_simple	1307.56	transactions_per_second	-143.01	0.901x
8	UDP_RR_Transaction_Rate	netperf_simple	4275.72	transactions_per_second	UDP_RR_Transaction_Rate	netperf_simple	4086.7	transactions_per_second	-189.02	0.956x
9	End to End Runtime	netperf_simple	398.570210934	seconds	End to End Runtime	netperf_simple	1798.07076192	seconds	1399.50055099	4.511x

Short diff

Removed the following keys which were expected to vary:

• run_uri
• sample_uri
• timestamp
• value

Sample 1

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'15.880',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 2

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'4.517',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 3

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'21.161',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 4

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'6.473',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 5

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'7.383',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 6

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'34.388',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 7

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'4.499',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Sample 8

	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'5.836',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',

Full diff

Removed the following keys which were expected to vary:

• run_uri
• sample_uri
• timestamp
• value

Sample 1

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'15.880',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'external',		6	u'ip_type': u'external',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_RR_Transaction_Rate',		14	u'metric': u'TCP_RR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 2

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'4.517',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_RR_Transaction_Rate',		14	u'metric': u'TCP_RR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 3

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'21.161',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_CRR_Transaction_Rate',		14	u'metric': u'TCP_CRR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 4

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'6.473',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_CRR_Transaction_Rate',		14	u'metric': u'TCP_CRR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 5

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'7.383',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_STREAM_Throughput',		14	u'metric': u'TCP_STREAM_Throughput',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'Mbits/sec'}		19	u'unit': u'Mbits/sec'}

Sample 6

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'34.388',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'TCP_STREAM_Throughput',		14	u'metric': u'TCP_STREAM_Throughput',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'Mbits/sec'}		19	u'unit': u'Mbits/sec'}

Sample 7

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'4.499',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'UDP_RR_Transaction_Rate',		14	u'metric': u'UDP_RR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 8

f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',	f	1	{'metadata': {u'client_machine_type': u'n1-standard-4',
	2	u'client_zone': u'us-central1-a',		2	u'client_zone': u'us-central1-a',
	3	u'cloud': u'GCP',		3	u'cloud': u'GCP',
t			t	4	u'conf_pct': u'5.836',
	4	u'image': u'debian-7-backports',		5	u'image': u'debian-7-backports',
	5	u'ip_type': u'internal',		6	u'ip_type': u'internal',
	6	u'machine_type': u'n1-standard-4',		7	u'machine_type': u'n1-standard-4',
	7	u'perfkitbenchmarker_version': u'unknown',		8	u'perfkitbenchmarker_version': u'unknown',
	8	u'receiving_zone': u'us-central1-a',		9	u'receiving_zone': u'us-central1-a',
	9	u'sending_zone': u'us-central1-a',		10	u'sending_zone': u'us-central1-a',
	10	u'server_machine_type': u'n1-standard-4',		11	u'server_machine_type': u'n1-standard-4',
	11	u'server_zone': u'us-central1-a',		12	u'server_zone': u'us-central1-a',
	12	u'zones': u'us-central1-a'},		13	u'zones': u'us-central1-a'},
	13	u'metric': u'UDP_RR_Transaction_Rate',		14	u'metric': u'UDP_RR_Transaction_Rate',
	14	u'official': False,		15	u'official': False,
	15	u'owner': u'connormccoy',		16	u'owner': u'connormccoy',
	16	u'product_name': u'PerfKitBenchmarker',		17	u'product_name': u'PerfKitBenchmarker',
	17	u'test': u'netperf_simple',		18	u'test': u'netperf_simple',
	18	u'unit': u'transactions_per_second'}		19	u'unit': u'transactions_per_second'}

Sample 9

t	1	{'metadata': {u'cloud': u'GCP',	t	1	{'metadata': {u'cloud': u'GCP',
	2	u'image': u'debian-7-backports',		2	u'image': u'debian-7-backports',
	3	u'machine_type': u'n1-standard-4',		3	u'machine_type': u'n1-standard-4',
	4	u'perfkitbenchmarker_version': u'unknown',		4	u'perfkitbenchmarker_version': u'unknown',
	5	u'zones': u'us-central1-a'},		5	u'zones': u'us-central1-a'},
	6	u'metric': u'End to End Runtime',		6	u'metric': u'End to End Runtime',
	7	u'official': False,		7	u'official': False,
	8	u'owner': u'connormccoy',		8	u'owner': u'connormccoy',
	9	u'product_name': u'PerfKitBenchmarker',		9	u'product_name': u'PerfKitBenchmarker',
	10	u'test': u'netperf_simple',		10	u'test': u'netperf_simple',
	11	u'unit': u'seconds'}		11	u'unit': u'seconds'}