[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [depictor/master] Big refactor



commit b30311728584e0682c3d02079932611d3d547e1a
Author: Tom <tom@xxxxxxxxx>
Date:   Thu Oct 27 14:34:41 2016 -0500

    Big refactor
    
    - Put in a stub for putting your own DirAuths in
    - Fix 20480 by lowercasing dirauth names consistenty
    - Fix 20464 by making all operations with vote_data smarter.
      - Insert with named columns
      - Examine table structure and add columns if a dirauth gets added
      - select named columns for the csv
    - Remove last hardcoded dirauth stuff from graphs and make the colors dynamic
    - Add a config option to ignore fallback directories (important for other networks)
    - Remove super-redundant dirauth searching in website.py
    - Move some stuff to a utility file
---
 .gitignore         |   4 +
 data/consensus.cfg |  17 +--
 graphs.py          | 380 +++++++++++++++++++++++++++--------------------------
 utility.py         |  86 ++++++++++++
 website.py         |  43 +++---
 write_website.py   | 191 ++++++++++-----------------
 6 files changed, 373 insertions(+), 348 deletions(-)

diff --git a/.gitignore b/.gitignore
index 45104d6..037b253 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,9 +2,13 @@
 *.p
 *.swp
 stem
+
 out/index.html
 out/graphs.html
 out/vote-stats.csv
 out/consensus-health*
 out/download-stats.csv
+out/fallback-dir-stats.csv
+out/historical.db
+
 data/historical.db
\ No newline at end of file
diff --git a/data/consensus.cfg b/data/consensus.cfg
index c4a2142..381684a 100644
--- a/data/consensus.cfg
+++ b/data/consensus.cfg
@@ -1,19 +1,6 @@
-# directory authorities for which to not provide data
+# control consensus-health output
 
-ignored_authorities turtles
-
-# directory authorities that run bandwidth scanners
-
-bandwidth_authorities moria1
-bandwidth_authorities gabelmoo
-bandwidth_authorities tor26
-bandwidth_authorities longclaw
-bandwidth_authorities maatuska
-
-# all current/historical bridge auths
-
-historical_bridge_authorities tonga
-historical_bridge_authorities bifrost
+ignore_fallback_authorities False
 
 # recognized tor consensus parameters
 
diff --git a/graphs.py b/graphs.py
index 9293b1f..6738a52 100755
--- a/graphs.py
+++ b/graphs.py
@@ -14,7 +14,7 @@ import stem.descriptor.remote
 from base64 import b64decode
 
 from website import WebsiteWriter
-from parseOldConsensuses import get_dirauths_in_tables
+from utility import get_dirauths, get_bwauths
 
 class GraphWriter(WebsiteWriter):
 	def write_website(self, filename):
@@ -65,13 +65,14 @@ class GraphWriter(WebsiteWriter):
 			+ "      text-align: center;\n"
 			+ "      display: none;\n"
 			+ "    }\n"
-			+ "    .faravahar {\n"
+
+			+ "    .auth1 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: steelblue;\n"
 			+ "      background-color: steelblue;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .gabelmoo, .orange {\n"
+			+ "    .auth2, .orange {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: orange;\n"
 			+ "      background-color: orange;\n"
@@ -80,13 +81,13 @@ class GraphWriter(WebsiteWriter):
 			+ "    .orange {\n"
 			+ "      fill: orange;"
 			+ "    }\n"
-			+ "    .moria1 {\n"
+			+ "    .auth3 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: yellow;\n"
 			+ "      background-color: yellow;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .maatuska, .green {\n"
+			+ "    .auth4, .green {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: green;\n"
 			+ "      background-color: green;\n"
@@ -95,7 +96,7 @@ class GraphWriter(WebsiteWriter):
 			+ "    .green {\n"
 			+ "      fill: green;"
 			+ "    }\n"
-			+ "    .longclaw, .red {\n"
+			+ "    .auth5, .red {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: red;\n"
 			+ "      background-color: red;\n"
@@ -104,36 +105,37 @@ class GraphWriter(WebsiteWriter):
 			+ "    .red {\n"
 			+ "      fill: red;"
 			+ "    }\n"
-			+ "    .tor26 {\n"
+			+ "    .auth6 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: purple;\n"
 			+ "      background-color: purple;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .urras {\n"
+			+ "    .auth7 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: black;\n"
 			+ "      background-color: black;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .turtles {\n"
+			+ "    .auth8 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: #0000FF;\n"
 			+ "      background-color: #0000FF;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .dizum {\n"
+			+ "    .auth9 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: limegreen;\n"
 			+ "      background-color: limegreen;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
-			+ "    .dannenberg {\n"
+			+ "    .auth10 {\n"
 			+ "      fill: none;\n"
 			+ "      stroke: pink;\n"
 			+ "      background-color: pink;\n"
 			+ "      stroke-width: 1.5px;\n"
 			+ "    }\n"
+
 			+ "  </style>\n"
 			+ "    <div class=\"center\">\n"
 			+ "      <div class=\"main-column\">\n"
@@ -159,6 +161,9 @@ class GraphWriter(WebsiteWriter):
 		"""
 		Write the graphs of the fallback directory mirrors
 		"""
+		if self.config['ignore_fallback_authorities']:
+			return
+
 		self.site.write("<br>\n\n\n"
 		+ " <!-- ================================================================= -->"
 		+ "<a name=\"fallbackdirgraphs\">\n"
@@ -167,8 +172,7 @@ class GraphWriter(WebsiteWriter):
 		+ "<br>\n"
 		+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
 		+ "  <colgroup>\n"
-		+ "    <col width=\"160\">\n"
-		+ "    <col width=\"640\">\n"
+		+ "    <col width=\"800\">\n"
 		+ "  </colgroup>\n"
 		+ "  <tr class=\"graphplaceholder\">\n"
 		+ "    <td>\n"
@@ -185,19 +189,17 @@ class GraphWriter(WebsiteWriter):
 		self.site.write("</table>\n")
 
 	#-----------------------------------------------------------------------------------------
-	def _write_number_of_relays_voted_about_graphs_spot(self, divName):
+	def _write_number_of_relays_voted_about_graphs_spot(self, divName, dirAuths):
 		self.site.write("  <tr>\n"
 		+ "    <td>\n"
-		+ "      <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
-		+ "        <span class=\"moria1\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Moria\n"
-		+ "        <span class=\"faravahar\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Faravahar\n"
-		+ "        <span class=\"gabelmoo\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Gabelmoo\n"
-		+ "        <span class=\"maatuska\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Maatuska\n"
-		+ "        <span class=\"longclaw\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Longclaw\n"
-		+ "        <span class=\"tor26\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> tor26\n"
-		+ "        <span class=\"dizum\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> dizum\n"
-		+ "        <span class=\"dannenberg\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> dannenberg\n"
-		+ "      </div>\n"
+		+ "      <div id=\"" + str(divName) + "\" class=\"graphbox\">\n")
+
+		i = 0
+		for d in dirAuths:
+			i += 1
+			self.site.write("        <span class=\"auth" + str(i) + "\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> " + d + "\n")
+
+		self.site.write("      </div>\n"
 		+ "    </td>\n"
 		+ "  </tr>\n")
 	def _write_number_of_relays_voted_about_graphs(self):
@@ -212,8 +214,7 @@ class GraphWriter(WebsiteWriter):
 		+ "<br>\n"
 		+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
 		+ "  <colgroup>\n"
-		+ "    <col width=\"160\">\n"
-		+ "    <col width=\"640\">\n"
+		+ "    <col width=\"800\">\n"
 		+ "  </colgroup>\n"
 		+ "  <tr class=\"graphplaceholder\">\n"
 		+ "    <td>\n"
@@ -222,31 +223,32 @@ class GraphWriter(WebsiteWriter):
 		+ "      </div>\n"
 		+ "    </td>\n"
 		+ "  </tr>\n")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_total_1")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_total_2")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_total_3")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_total_4")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_running_1")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_running_2")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_running_3")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_running_4")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3")
-		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4")
+		self._write_number_of_relays_voted_about_graphs_spot("voted_total_1", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_total_2", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_total_3", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_total_4", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_running_1", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_running_2", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_running_3", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_running_4", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_1", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_2", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_3", get_dirauths())
+		self._write_number_of_relays_voted_about_graphs_spot("voted_notrunning_4", get_dirauths())
 		self.site.write("</table>\n")
 
 	#-----------------------------------------------------------------------------------------
-	def _write_bandwidth_scanner_graphs_spot(self, divName):
+	def _write_bandwidth_scanner_graphs_spot(self, divName, bwAuths):
 		self.site.write("  <tr>\n"
 		+ "    <td>\n"
-		+ "      <div id=\"" + str(divName) + "\" class=\"graphbox\">\n"
-		+ "        <span class=\"moria1\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Moria\n"
-		+ "        <span class=\"faravahar\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Faravahar\n"
-		+ "        <span class=\"gabelmoo\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Gabelmoo\n"
-		+ "        <span class=\"maatuska\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Maatuska\n"
-		+ "        <span class=\"longclaw\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> Longclaw\n"
-		+ "      </div>\n"
+		+ "      <div id=\"" + str(divName) + "\" class=\"graphbox\">\n")
+
+		i = 0
+		for d in bwAuths:
+			i += 1
+			self.site.write("        <span class=\"auth" + str(i) + "\" style=\"margin-left:5px\">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span> " + d + "\n")
+
+		self.site.write("      </div>\n"
 		+ "    </td>\n"
 		+ "  </tr>\n")
 	def _write_bandwidth_scanner_graphs(self):
@@ -261,8 +263,7 @@ class GraphWriter(WebsiteWriter):
 		+ "<br>\n"
 		+ "<table border=\"0\" cellpadding=\"4\" cellspacing=\"0\" summary=\"\">\n"
 		+ "  <colgroup>\n"
-		+ "    <col width=\"160\">\n"
-		+ "    <col width=\"640\">\n"
+		+ "    <col width=\"800\">\n"
 		+ "  </colgroup>\n"
 		+ "  <tr class=\"graphplaceholder\">\n"
 		+ "    <td>\n"
@@ -271,10 +272,10 @@ class GraphWriter(WebsiteWriter):
 		+ "      </div>\n"
 		+ "    </td>\n"
 		+ "  </tr>\n")
-		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1")
-		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2")
-		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3")
-		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4")
+		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_1", get_bwauths())
+		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_2", get_bwauths())
+		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_3", get_bwauths())
+		self._write_bandwidth_scanner_graphs_spot("bwauth_measured_4", get_bwauths())
 		#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_1")
 		#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_2")
 		#self._write_bandwidth_scanner_graphs_spot("bwauth_running_unmeasured_3")
@@ -289,10 +290,9 @@ class GraphWriter(WebsiteWriter):
 		    HEIGHT = 500,
 		    MARGIN = {top: 40, right: 40, bottom: 40, left: 40};
 
-		var bwauths = ["faravahar","gabelmoo","moria1","maatuska","longclaw"];
-		var dirauths = """ + str(get_dirauths_in_tables()) + """;
-		dirauths.splice(dirauths.indexOf('urras'), 1);
-		dirauths.splice(dirauths.indexOf('turtles'), 1);
+		var bwauths = """ + str(get_bwauths().keys()) + """;
+		var dirauths = """ + str(get_dirauths().keys()) + """;
+		var ignore_fallback_dirs = """ + str(self.config['ignore_fallback_authorities']).lower() + """;
 
 		var _getBandwidthDataValue = function(d, dirauth) { return d[dirauth + "_bwauth"]; }
 		var _getRunningDataValue = function(d, dirauth) { return d[dirauth + "_running"]; }
@@ -360,7 +360,7 @@ class GraphWriter(WebsiteWriter):
 	    ];
 
 	    relays_done = false;
-	    fallbackdirs_done = false;
+	    fallbackdirs_done = ignore_fallback_dirs;
 		fetch("vote-stats.csv").then(function(response) {
 			return response.text();
 		}).then(function(text) {
@@ -381,11 +381,11 @@ class GraphWriter(WebsiteWriter):
 			graph = GRAPHS_TO_GENERATE[g];
 
 			if(graph.data_slice+1 > data.length) {
-				data_subset = data.slice(1);
+				data_subset = data.slice(0);
 				console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
 			}
 			else
-				data_subset = data.slice(1, graph.data_slice+1);
+				data_subset = data.slice(0, graph.data_slice+1);
 			data_subset.reverse();
 
 			// Calculate the Graph Boundaries -----------------------------------------
@@ -436,11 +436,12 @@ class GraphWriter(WebsiteWriter):
 				.domain([avg-(5*stddev), avg+(5*stddev)])
 			    .range([HEIGHT, 0]);
 
+			var i = 1;
 			var lines = []
 			for(auth in graph.authorities)
 			{
 				this_auth = graph.authorities[auth];
-				lines.push({auth: this_auth, line: (function(dirAuthClosure) {
+				lines.push({authIndex: i, line: (function(dirAuthClosure) {
 					return d3.line()
 					    .defined(function(d) { 
 						return d && graph.data_func(d, dirAuthClosure) && 
@@ -449,6 +450,7 @@ class GraphWriter(WebsiteWriter):
 			    		.x(function(d) { return x(d.date); })
 				    	.y(function(d) { return y(graph.data_func(d, dirAuthClosure)); });
 				    })(this_auth)});
+			    i++;
 			}
 
 			var svg = d3.select("#" + graph.div).append("svg")
@@ -470,7 +472,7 @@ class GraphWriter(WebsiteWriter):
 			for(l in lines)
 			{
 				svg.append("path")
-			    	.attr("class", lines[l].auth)
+			    	.attr("class", "auth" + lines[l].authIndex)
 				    .attr("d", lines[l].line);
 			}
 
@@ -496,136 +498,139 @@ class GraphWriter(WebsiteWriter):
 
 		});
 
-		fetch("fallback-dir-stats.csv").then(function(response) {
-			return response.text();
-		}).then(function(text) {
-			return d3.csvParse(text, function(d) {
-				for(i in d) {
-					if(i == "date")
-						d[i] = new Date(Number(d[i]));
+		if(!ignore_fallback_dirs) {
+
+			fetch("fallback-dir-stats.csv").then(function(response) {
+				return response.text();
+			}).then(function(text) {
+				return d3.csvParse(text, function(d) {
+					for(i in d) {
+						if(i == "date")
+							d[i] = new Date(Number(d[i]));
+						else
+							d[i] = Number(d[i]);
+					}
+					return d;
+				});
+			}).then(function(data) {
+				var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
+				/*Pie Graph
+				data_subset = data.slice(0);
+				data_subset = [
+					{'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
+					{'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
+					{'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
+				];
+				var data_func = function(d) { return d.value; };
+				var arcs = d3.pie()
+					.sort(null)
+					.value(data_func)(data_subset);
+
+				var svg = d3.select('#fallbackdirs_pie')
+					.append('svg')
+					.attr('width', WIDTH)
+					.attr('height', HEIGHT)
+					.append('g')
+					.attr('transform', 'translate(' + (WIDTH / 2) +	',' + (HEIGHT / 2) + ')');
+
+				var arc = d3.arc()
+					.innerRadius(0)
+					.outerRadius(100);
+
+				var path = svg.selectAll('path')
+					.data(arcs)
+					.enter()
+					.append('path')
+					.attr('d', arc)
+					.attr('class', function(d, i) {
+						return key_to_color(d.data.label);
+					});*/
+
+				//Line Graphs
+				for(g in FALLBACK_GRAPHS_TO_GENERATE)
+				{
+					graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+
+					if(graph.data_slice+1 > data.length) {
+						data_subset = data.slice(0);
+						console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
+					}
 					else
-						d[i] = Number(d[i]);
-				}
-				return d;
-			});
-		}).then(function(data) {
-			var key_to_color = function(k) { return k == 'fallback_dirs_running' ? 'green' : k == 'fallback_dirs_notrunning' ? 'orange' : 'red' };
-			/*Pie Graph
-			data_subset = data.slice(0);
-			data_subset = [
-				{'label' : 'fallback_dirs_running', 'value': data_subset[0]['fallback_dirs_running']},
-				{'label' : 'fallback_dirs_notrunning', 'value': data_subset[0]['fallback_dirs_notrunning']},
-				{'label' : 'fallback_dirs_missing', 'value': data_subset[0]['fallback_dirs_missing']},
-			];
-			var data_func = function(d) { return d.value; };
-			var arcs = d3.pie()
-				.sort(null)
-				.value(data_func)(data_subset);
-
-			var svg = d3.select('#fallbackdirs_pie')
-				.append('svg')
-				.attr('width', WIDTH)
-				.attr('height', HEIGHT)
-				.append('g')
-				.attr('transform', 'translate(' + (WIDTH / 2) +	',' + (HEIGHT / 2) + ')');
-
-			var arc = d3.arc()
-				.innerRadius(0)
-				.outerRadius(100);
-
-			var path = svg.selectAll('path')
-				.data(arcs)
-				.enter()
-				.append('path')
-				.attr('d', arc)
-				.attr('class', function(d, i) {
-					return key_to_color(d.data.label);
-				});*/
-
-			//Line Graphs
-			for(g in FALLBACK_GRAPHS_TO_GENERATE)
-			{
-				graph = FALLBACK_GRAPHS_TO_GENERATE[g];
+						data_subset = data.slice(0, graph.data_slice);
+					data_subset.reverse();
+				
+					max = 0
+					for(d in data_subset) {
+						x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
+						if(x > max)
+							max = x;
+					}
 
-				if(graph.data_slice+1 > data.length) {
-					data_subset = data.slice(0);
-					console.log("("+graph.title+") Requested " + (graph.data_slice+1) + " but there are only " + data.length + " items...");
-				}
-				else
-					data_subset = data.slice(0, graph.data_slice);
-				data_subset.reverse();
-			
-				max = 0
-				for(d in data_subset) {
-					x = data_subset[d]['fallback_dirs_running'] + data_subset[d]['fallback_dirs_notrunning'] + data_subset[d]['fallback_dirs_missing'];
-					if(x > max)
-						max = x;
+					var x = d3.scaleTime()
+						.domain([data_subset[0].date, data_subset[data_subset.length-1].date])
+						.range([0, WIDTH]);
+
+					var y = d3.scaleLinear()
+						.domain([0, max])
+						.range([HEIGHT, 0]);
+
+					var stack = d3.stack()
+						.keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
+						.order(d3.stackOrderNone)
+						.offset(d3.stackOffsetNone);
+
+					var area = d3.area()
+						.x(function(d, i) { return x(d.data.date); })
+						.y0(function(d) { return y(d[0]); })
+						.y1(function(d) { return y(d[1]); });
+
+					var svg = d3.select("#" + graph.div).append("svg")
+						.attr("width", WIDTH + MARGIN.left + MARGIN.right)
+						.attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
+						.append("g")
+						.attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
+
+					var layer = svg.selectAll(".layer")
+						.data(stack(data_subset))
+						.enter().append("g")
+						//.attr("class", "layer");
+
+					layer.append("path")
+						//.attr("class", "area")
+						.attr("class", function(d) { return key_to_color(d.key); })
+						.attr("d", area);
+
+					svg.append("g")
+						.attr("class", "axis axis--x")
+						.attr("transform", "translate(0," + HEIGHT + ")")
+						.call(d3.axisBottom().scale(x));
+
+					svg.append("g")
+						.attr("class", "axis axis--y")
+						.call(d3.axisLeft().scale(y));
+
+					svg.append("text")
+						.attr("x", (WIDTH / 2))
+						.attr("y", 0 - (MARGIN.top / 2))
+						.attr("text-anchor", "middle")
+						.attr("class", "graph-title")
+						.text(graph.title);
 				}
 
-				var x = d3.scaleTime()
-					.domain([data_subset[0].date, data_subset[data_subset.length-1].date])
-					.range([0, WIDTH]);
-
-				var y = d3.scaleLinear()
-					.domain([0, max])
-					.range([HEIGHT, 0]);
-
-				var stack = d3.stack()
-					.keys(["fallback_dirs_missing", "fallback_dirs_notrunning", "fallback_dirs_running"])
-					.order(d3.stackOrderNone)
-					.offset(d3.stackOffsetNone);
-
-				var area = d3.area()
-					.x(function(d, i) { return x(d.data.date); })
-					.y0(function(d) { return y(d[0]); })
-					.y1(function(d) { return y(d[1]); });
-
-				var svg = d3.select("#" + graph.div).append("svg")
-					.attr("width", WIDTH + MARGIN.left + MARGIN.right)
-					.attr("height", HEIGHT + MARGIN.top + MARGIN.bottom)
-					.append("g")
-					.attr("transform", "translate(" + MARGIN.left + "," + MARGIN.top + ")");
-
-				var layer = svg.selectAll(".layer")
-					.data(stack(data_subset))
-					.enter().append("g")
-					//.attr("class", "layer");
-
-				layer.append("path")
-					//.attr("class", "area")
-					.attr("class", function(d) { return key_to_color(d.key); })
-					.attr("d", area);
-
-				svg.append("g")
-					.attr("class", "axis axis--x")
-					.attr("transform", "translate(0," + HEIGHT + ")")
-					.call(d3.axisBottom().scale(x));
-
-				svg.append("g")
-					.attr("class", "axis axis--y")
-					.call(d3.axisLeft().scale(y));
-
-				svg.append("text")
-					.attr("x", (WIDTH / 2))
-					.attr("y", 0 - (MARGIN.top / 2))
-					.attr("text-anchor", "middle")
-					.attr("class", "graph-title")
-					.text(graph.title);
-			}
-
-			
-			fallbackdirs_done = true;
-			if(relays_done) {
-				var toShow = document.getElementsByClassName('graphbox');
-				for(i=0; i<toShow.length; i++) {
-					toShow[i].style.display = 'block';
-				}
-				var toHide = document.getElementsByClassName('graphplaceholder');
-				for(i=0; i<toHide.length; i++) {
-					toHide[i].style.display = 'none';
+				
+				fallbackdirs_done = true;
+				if(relays_done) {
+					var toShow = document.getElementsByClassName('graphbox');
+					for(i=0; i<toShow.length; i++) {
+						toShow[i].style.display = 'block';
+					}
+					var toHide = document.getElementsByClassName('graphplaceholder');
+					for(i=0; i<toHide.length; i++) {
+						toHide[i].style.display = 'none';
+					}
 				}
-			}
-		});
+			});
+		}
 
 		</script>"""
 		self.site.write(s)
@@ -654,9 +659,8 @@ if __name__ == '__main__':
 
 
 	CONFIG = stem.util.conf.config_dict('consensus', {
-                                    'ignored_authorities': [],
-                                    'bandwidth_authorities': [],
                                     'known_params': [],
+                                    'ignore_fallback_authorities': False,
                                     })
 	config = stem.util.conf.get_config("consensus")
 	config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
diff --git a/utility.py b/utility.py
new file mode 100644
index 0000000..6cebe6b
--- /dev/null
+++ b/utility.py
@@ -0,0 +1,86 @@
+import time
+import datetime
+
+import stem.descriptor
+import stem.descriptor.remote
+import stem.util.conf
+import stem.util.enum
+
+from stem.util.lru_cache import lru_cache
+
+@lru_cache()
+def get_dirauths():
+	#Remove any BridgeAuths
+	return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.v3ident)
+
+@lru_cache()
+def get_bwauths():
+	return dict((k.lower(), v) for (k, v) in stem.descriptor.remote.get_authorities().items() if v.is_bandwidth_authority)
+
+downloader = stem.descriptor.remote.DescriptorDownloader(
+	timeout = 60,
+	fall_back_to_authority = False,
+	document_handler = stem.descriptor.DocumentHandler.DOCUMENT,
+)
+
+def get_consensuses():
+	"""
+	Provides a mapping of directory authority nicknames to their present consensus.
+
+	:returns: tuple of the form ({authority => consensus}, issues, runtimes)
+	"""
+
+	return _get_documents('consensus', '/tor/status-vote/current/consensus')
+
+
+def get_votes():
+	"""
+	Provides a mapping of directory authority nicknames to their present vote.
+
+	:returns: tuple of the form ({authority => vote}, issues, runtimes)
+	"""
+
+	return _get_documents('vote', '/tor/status-vote/current/authority')
+
+
+def _get_documents(label, resource):
+	documents, issues, runtimes = {}, [], {}
+
+	for (nickname, authority) in get_dirauths().items():
+		if authority.v3ident is None:
+			continue	# not a voting authority
+
+		query = downloader.query(
+			resource,
+			endpoints = [(authority.address, authority.dir_port)],
+			default_params = False,
+		)
+
+		try:
+			start_time = time.time()
+			documents[nickname] = query.run()[0]
+			runtimes[nickname] = time.time() - start_time
+		except Exception, exc:
+			if label == 'vote':
+				# try to download the vote via the other authorities
+
+				v3ident = authority.v3ident
+
+				query = downloader.query(
+					'/tor/status-vote/current/%s' % v3ident,
+					default_params = False,
+				)
+
+				query.run(True)
+
+				if not query.error:
+					documents[nickname] = list(query)[0]
+					continue
+
+			issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc))
+
+	return documents, issues, runtimes
+
+def unix_time(dt):
+    return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
+
diff --git a/website.py b/website.py
index 7b79ec9..592ad0e 100755
--- a/website.py
+++ b/website.py
@@ -10,20 +10,23 @@ import os
 import time
 import operator
 import datetime
-import stem.descriptor.remote
+
 from base64 import b64decode
 from Crypto.PublicKey import RSA
 
+import stem.descriptor.remote
+
+from utility import get_dirauths, get_bwauths
+
 class WebsiteWriter:
 	consensus = None
 	votes = None
 	fallback_dirs = None
-	config_set = False
 	known_authorities = []
-	historical_bridge_authorities = []
 	bandwidth_authorities = []
 	consensus_expirey = datetime.timedelta(hours=3)
 	directory_key_warning_time = datetime.timedelta(days=14)
+	config = {}
 	known_params = []
 	def write_website(self, filename, include_relay_info=True):
 		self.site = open(filename, 'w')
@@ -49,13 +52,10 @@ class WebsiteWriter:
 		self.site.close()
 
 	def set_consensuses(self, c):
-		if not self.config_set:
-			raise Exception("Set config before calling")
 		self.consensuses = c
 		self.consensus = max(c.itervalues(), key=operator.attrgetter('valid_after'))
-		self.known_authorities = set([r.nickname for r in self.consensus.routers.values() if 'Authority' in r.flags and r.nickname not in self.historical_bridge_authorities])
-		self.known_authorities.update([r.nickname for r in self.consensus.directory_authorities])
-		self.known_authorities.update([r for r in stem.descriptor.remote.get_authorities().keys() if r not in self.historical_bridge_authorities])
+		self.known_authorities = get_dirauths().keys()
+		self.bandwidth_authorities = get_bwauths().keys()
 	def set_votes(self, v):
 		self.votes = v
 	def set_consensus_expirey(self, timedelta):
@@ -63,10 +63,8 @@ class WebsiteWriter:
 	def set_directory_key_warning_time(self, timedelta):
 		self.directory_key_warning_time = timedelta
 	def set_config(self, config):
-		self.config_set = True
+		self.config = config
 		self.known_params = config['known_params']
-		self.historical_bridge_authorities = config['historical_bridge_authorities']
-		self.bandwidth_authorities = config['bandwidth_authorities']
 	def set_fallback_dirs(self, fallback_dirs):
 		self.fallback_dirs = fallback_dirs
 	def get_consensus_time(self):
@@ -171,9 +169,9 @@ class WebsiteWriter:
 			+ "    <td>" + dirauth_nickname + "</td>\n")
 			
 			#Try and find a structure that has it's IP & Port
-			authority = [r for r in self.consensus.routers.values() if r.nickname == dirauth_nickname and 'Authority' in r.flags]
+			authority = [r for r in self.consensus.routers.values() if r.nickname.lower() == dirauth_nickname and 'Authority' in r.flags]
 			if not authority:
-				authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname]
+				authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname]
 			if authority:
 				authority = authority[0]
 				self.site.write("    <td><a href=\"http://"; + authority.address + ":" + str(authority.dir_port)
@@ -183,20 +181,20 @@ class WebsiteWriter:
 			else:
 				self.site.write("    <td colspan=\"2\" class=\"oiv\">Missing entirely from consensus</td>\n")
 				
-			if dirauth_nickname in [d.nickname for d in self.consensus.directory_authorities]:
+			if dirauth_nickname in [d.nickname.lower() for d in self.consensus.directory_authorities]:
 				#The above structure is sufficient for getting the address & port
 				# but we need this structure for the authority's fingerprint
-				authority = [d for d in self.consensus.directory_authorities if d.nickname == dirauth_nickname][0]
+				authority = [d for d in self.consensus.directory_authorities if d.nickname.lower() == dirauth_nickname][0]
 				if authority.fingerprint in signingFPs:
 					self.site.write("    <td>" + signingFPs[authority.fingerprint] + "</td>\n")
-				elif authority.nickname in self.consensuses:
+				elif authority.nickname.lower() in self.consensuses:
 					self.site.write("    <td class=\"oiv\">Missing Signature! "
 					+ "Valid-after time of auth's displayed consensus: "
-					+ self.consensuses[authority.nickname].valid_after.isoformat().replace("T", " ")
+					+ self.consensuses[authority.nickname.lower()].valid_after.isoformat().replace("T", " ")
 					+ "</td>\n")
 				else:
 					self.site.write("    <td class=\"oiv\">Missing Signature, and "
-					+ authority.nickname + " does not have a consensus available</td>\n")
+					+ authority.nickname.lower() + " does not have a consensus available</td>\n")
 			self.site.write("  </tr>\n")
 		self.site.write("</table>\n")
 
@@ -592,6 +590,9 @@ class WebsiteWriter:
 		"""
 		Write the status of the fallback directory mirrors
 		"""
+		if self.config['ignore_fallback_authorities']:
+				return
+
 		self.site.write("<br>\n\n\n"
 		+ " <!-- ================================================================= -->"
 		+ "<a name=\"fallbackdirstatus\">\n"
@@ -649,7 +650,7 @@ class WebsiteWriter:
 		+ "Authority versions</a></h3>\n"
 		+ "<br>\n")
 
-		authorityVersions = [(r.nickname, r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags]
+		authorityVersions = [(r.nickname.lower(), r.version) for r in self.consensus.routers.values() if 'Authority' in r.flags]
 		if not authorityVersions:
 			self.site.write("<p>(No relays with Authority flag found.)</p>\n")
 		else:
@@ -683,6 +684,7 @@ class WebsiteWriter:
 		downloadData = {}
 		for l in lines:
 			parts = l.split(',')
+			parts[0] = parts[0].lower()
 			if int(parts[1]) < cutoff:
 				continue
 			if parts[0] not in downloadData:
@@ -1071,9 +1073,8 @@ if __name__ == '__main__':
 		
 
 	CONFIG = stem.util.conf.config_dict('consensus', {
-                                    'ignored_authorities': [],
-                                    'bandwidth_authorities': [],
                                     'known_params': [],
+                                    'ignore_fallback_authorities': False,
                                     })
 	config = stem.util.conf.get_config("consensus")
 	config.load(os.path.join(os.path.dirname(__file__), 'data', 'consensus.cfg'))
diff --git a/write_website.py b/write_website.py
index 7216d03..3d4760d 100755
--- a/write_website.py
+++ b/write_website.py
@@ -20,36 +20,35 @@ import stem.descriptor.remote
 import stem.util.conf
 import stem.util.enum
 
-from stem import Flag
-from stem.util.lru_cache import lru_cache
+from stem.descriptor.remote import FallbackDirectory
+from stem.descriptor.remote import DirectoryAuthority
 
+from utility import *
 from website import WebsiteWriter
 from graphs import GraphWriter
-from parseOldConsensuses import get_dirauths_in_tables
 
-DIRECTORY_AUTHORITIES = stem.descriptor.remote.get_authorities()
+
+#If you're running your own test network, you define your DirAuths here
+# dir-source line: dir-source authority_name v3ident hostname ip  DirPort  OrPort
+# r line: r nickname base64(fingerprint + "=")   -> python -c "x = ''; import sys; import base64; sys.stdout.write(''.join('{:02x}'.format(ord(c)) for c in base64.b64decode(x)))"
+
+#stem.descriptor.remote.DIRECTORY_AUTHORITIES = {
+#'Faravahar': DirectoryAuthority(
+#    nickname = 'Faravahar',
+#    address = '154.35.175.225',
+#    or_port = 443,
+#    dir_port = 80,
+#    is_bandwidth_authority = True,
+#    fingerprint = 'CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC',
+#    v3ident = 'EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97',
+#  ),	
+#}
 
 CONFIG = stem.util.conf.config_dict('consensus', {
-	'ignored_authorities': [],
-	'bandwidth_authorities': [],
 	'known_params': [],
-	'historical_bridge_authorities' : []
+	'ignore_fallback_authorities' : False
 })
 
-downloader = stem.descriptor.remote.DescriptorDownloader(
-	timeout = 60,
-	fall_back_to_authority = False,
-	document_handler = stem.descriptor.DocumentHandler.DOCUMENT,
-)
-
-
-@lru_cache()
-def directory_authorities():
-	return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not in CONFIG['ignored_authorities'])
-
-def unix_time(dt):
-    return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000.0
-
 def main():
 	# loads configuration data
 	config = stem.util.conf.get_config("consensus")
@@ -65,47 +64,54 @@ def main():
 	f.close()
 
 	# Calculate the fallback directory info
-	from stem.descriptor.remote import FallbackDirectory
-	fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote()
+	if not CONFIG['ignore_fallback_authorities']:
+		fallback_dirs = stem.descriptor.remote.FallbackDirectory.from_remote()
+	else:
+		fallback_dirs = []
+	
 	# great for debugging
 	#import pickle
+	#pickle.dump(consensuses, open('consensus.p', 'wb'))
+	#pickle.dump(votes, open('votes.p', 'wb'))
 	#pickle.dump(fallback_dirs, open('fallback_dirs.p', 'wb'))
 
-	# Calculate the number of fallback directory authorities present in the consensus and insert it into the database
-	fallback_dirs_running = 0
-	fallback_dirs_notrunning = 0
-	for relay_fp in consensuses.values()[0].routers:
-		if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
-			fallback_dirs_running += 1
-		elif relay_fp in fallback_dirs:
-			fallback_dirs_notrunning += 1
-				
-	insertValues = [unix_time(consensuses.values()[0].valid_after)]
-	insertValues.append(fallback_dirs_running)
-	insertValues.append(fallback_dirs_notrunning)
-	insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
 
 	dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
 
-	dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
-	dbc.commit()
+	# Calculate the number of fallback directory authorities present in the consensus and insert it into the database
+	if not CONFIG['ignore_fallback_authorities']:
+		fallback_dirs_running = 0
+		fallback_dirs_notrunning = 0
+		for relay_fp in consensuses.values()[0].routers:
+			if relay_fp in fallback_dirs and 'Running' in consensuses.values()[0].routers[relay_fp].flags:
+				fallback_dirs_running += 1
+			elif relay_fp in fallback_dirs:
+				fallback_dirs_notrunning += 1
+					
+		insertValues = [unix_time(consensuses.values()[0].valid_after)]
+		insertValues.append(fallback_dirs_running)
+		insertValues.append(fallback_dirs_notrunning)
+		insertValues.append(len(fallback_dirs) - fallback_dirs_running - fallback_dirs_notrunning)
+
+		dbc.execute("CREATE TABLE IF NOT EXISTS fallback_dir_data (date integer, fallback_dirs_running integer, fallback_dirs_notrunning integer, fallback_dirs_missing integer, PRIMARY KEY(date ASC));")
+		dbc.commit()
 
-	dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
-	dbc.commit()
+		dbc.execute("INSERT OR REPLACE INTO fallback_dir_data VALUES (?,?,?,?)", insertValues)
+		dbc.commit()
 
-	# Write out the updated csv file for the graphs
-	fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
-	f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
-	f.write("date")
-	f.write(",fallback_dirs_running")
-	f.write(",fallback_dirs_notrunning")
-	f.write(",fallback_dirs_missing")
-	f.write("\n")
-	for r in fallback_dir_data.fetchall():
-		for v in r:
-			f.write(("0" if v == None else str(v)) + ",")
+		# Write out the updated csv file for the graphs
+		fallback_dir_data = dbc.execute("SELECT * from fallback_dir_data ORDER BY date DESC LIMIT 2160")
+		f = open(os.path.join(os.path.dirname(__file__), 'out', 'fallback-dir-stats.csv'), 'w')
+		f.write("date")
+		f.write(",fallback_dirs_running")
+		f.write(",fallback_dirs_notrunning")
+		f.write(",fallback_dirs_missing")
 		f.write("\n")
-	f.close()
+		for r in fallback_dir_data.fetchall():
+			for v in r:
+				f.write(("0" if v == None else str(v)) + ",")
+			f.write("\n")
+		f.close()
 
 	# Calculate the number of known and measured relays for each dirauth and insert it into the database
 	data = {}
@@ -130,10 +136,8 @@ def main():
 	createColumns = ""
 	insertColumns = "date"
 	insertQuestions = ""
-	import pdb
-	pdb.set_trace()
-	for dirauth_nickname in directory_authorities():
-		dirauth_nickname = dirauth_nickname.lower()
+
+	for dirauth_nickname in get_dirauths():
 		if vote_data_columns and dirauth_nickname not in vote_data_columns:
 			dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_known integer")
 			dbc.execute("ALTER TABLE vote_data ADD COLUMN " + dirauth_nickname + "_running integer")
@@ -155,12 +159,15 @@ def main():
 	dbc.commit()
 
 	# Write out the updated csv file for the graphs
+	vote_data_columns = []
+	vote_data_schema = dbc.execute("PRAGMA table_info(vote_data)")
+	for c in vote_data_schema:
+		vote_data_columns.append(c[1])
+
 	vote_data = dbc.execute("SELECT * from vote_data ORDER BY date DESC LIMIT 2160")
 	f = open(os.path.join(os.path.dirname(__file__), 'out', 'vote-stats.csv'), 'w')
-	f.write("date")
-	for d in get_dirauths_in_tables():
-		s = "," + d + "_known," + d + "_running," + d + "_bwauth"
-		f.write(s)
+	for c in vote_data_columns:
+		f.write(c + ",")
 	f.write("\n")
 	for r in vote_data.fetchall():
 		for v in r:
@@ -168,11 +175,6 @@ def main():
 		f.write("\n")
 	f.close()
 
-	# great for debugging
-	#import pickle
-	#pickle.dump(consensuses, open('consensus.p', 'wb'))
-	#pickle.dump(votes, open('votes.p', 'wb'))
-
 	# produces the website
 	w = WebsiteWriter()
 	w.set_config(CONFIG)
@@ -220,65 +222,6 @@ def main():
 				os.remove(os.path.join(os.path.dirname(__file__), 'out', f))
 
 
-def get_consensuses():
-	"""
-	Provides a mapping of directory authority nicknames to their present consensus.
-
-	:returns: tuple of the form ({authority => consensus}, issues, runtimes)
-	"""
-
-	return _get_documents('consensus', '/tor/status-vote/current/consensus')
-
-
-def get_votes():
-	"""
-	Provides a mapping of directory authority nicknames to their present vote.
-
-	:returns: tuple of the form ({authority => vote}, issues, runtimes)
-	"""
-
-	return _get_documents('vote', '/tor/status-vote/current/authority')
-
-
-def _get_documents(label, resource):
-	documents, issues, runtimes = {}, [], {}
-
-	for authority in directory_authorities().values():
-		if authority.v3ident is None:
-			continue	# not a voting authority
-
-		query = downloader.query(
-			resource,
-			endpoints = [(authority.address, authority.dir_port)],
-			default_params = False,
-		)
-
-		try:
-			start_time = time.time()
-			documents[authority.nickname] = query.run()[0]
-			runtimes[authority.nickname] = time.time() - start_time
-		except Exception, exc:
-			if label == 'vote':
-				# try to download the vote via the other authorities
-
-				v3ident = directory_authorities()[authority.nickname].v3ident
-
-				query = downloader.query(
-					'/tor/status-vote/current/%s' % v3ident,
-					default_params = False,
-				)
-
-				query.run(True)
-
-				if not query.error:
-					documents[authority.nickname] = list(query)[0]
-					continue
-
-			issues.append(('AUTHORITY_UNAVAILABLE', label, authority, query.download_url, exc))
-
-	return documents, issues, runtimes
-
-
 if __name__ == '__main__':
 	try:
 		main()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits