viz and coloring nodes by cluster label

Author: xiaohan2012

clustering.py

@@ -11,7 +11,7 @@ def greedy_clustering_on_graph(
         metric=string_similar_probability,
         threshold=0.8):
     def get_text(n):
-        return '{} {}'.format(
+        return u'{} {}'.format(
             g.node[n]['subject'], g.node[n]['body']
         )
     cluster_assignment = {}

dump_events_to_json.py

@@ -40,13 +40,14 @@ def run(candidate_tree_path,
         assignment = greedy_clustering_on_graph(e)
         for n in e.nodes_iter():
             e.node[n]['cluster_label'] = assignment[n]
-
+            
     if to_original_graph:
         events = map(convert_to_original_graph,
                      events)
 
     d3_events = [to_d3_graph(e)
                  for e in events]
+
     json_dump(d3_events, output_path)
 
 

html/js/main.js

@@ -25,8 +25,9 @@ $(document).ready(function(){
 	var palette = d3.scale.ordinal()
 		.domain([EDGE_BROADCAST, EDGE_REPLY, EDGE_RELAY])
 		.range(d3.scale.category10().range());
-	var format_time = d3.time.format("%Y-%m-%d");
+	var cluster_palette = d3.scale.category20c();
 
+	var format_time = d3.time.format("%Y-%m-%d");
 
 	function get_config(mc){
 		var url_dict = {
@@ -86,7 +87,7 @@ $(document).ready(function(){
 			},
 			'meta_graph': {
 				svg: {width: 1280, height: 1500},
-				force: {charge: -1000, linkDistance: 150},
+				force: {charge: -500, linkDistance: 50},
 				tip: {
 					html: function(d){
 						console.log('iteraction:', d);
@@ -96,11 +97,13 @@ $(document).ready(function(){
 						d['recipients_str'] = _.map(d['recipients'], function(r){
 							return r.name;
 						}).join(',  ');
-						return dict2html(d, ['subject', 'body', 'sender_str', 'recipients_str', 'date', 'message_id']);
+						return dict2html(d, ['subject', 'body', 'hashtags', 'sender_str', 'recipients_str', 'date', 'message_id']);
 					}
 				},
 				node: {
-					fill: 'red',
+					fill: function(d){
+						return cluster_palette(d['cluster_label']);
+					},
 					r: 8,
 					label: dataset_setting.node_label
 				},
@@ -174,7 +177,6 @@ $(document).ready(function(){
 			mc,
 			url_dict
 		);
-		console.log('ret["force"]:', ret['force']);
 		var charge_from_input = parseInt($("#charge").val());
 		if(charge_from_input){
 			ret.force.charge = charge_from_input;

html/js/util.js

@@ -40,6 +40,10 @@ function init_dataset_and_paths_widget(paths_json_name){
 	$('#dataset').on('change', function(){
 		d3.json("data/" + $(this).val() + "/" + paths_json_name,
 				function(error, result_paths) {
+					if(error != null){
+						console.log(error);
+					}
+					
 					$('#dataPathSelection').children().remove();
 					_.each(result_paths.sort(), function(p, index){
 						var opt;

html/js/viz.js

@@ -91,10 +91,12 @@ function load_event_1(config){
 			.on('mouseover', tip.show)
 			.on('mouseout',  tip.hide)
 
-		var node_labels = gnodes.append("text")
-			.text(config.node.label)
-			.attr('font-size', 10)
-			.attr('font-weight', 'bold');
+		if(false){
+			var node_labels = gnodes.append("text")
+				.text(config.node.label)
+				.attr('font-size', 10)
+				.attr('font-weight', 'bold');
+		}
 
 		force.on("tick", function() {
 			link.attr("x1", function(d) { return d.source.x; })
@@ -117,147 +119,3 @@ function load_event_1(config){
 		});
 	});
 }
-
-
-
-// ######## DEPRECATED ########
-function load_event(data_path, kth){
-	var width = 960,
-	height = 1000;
-
-	var EDGE_BROADCAST = 1, EDGE_REPLY = 2, EDGE_RELAY = 3;
-	var palette = d3.scale.ordinal()
-		.domain([EDGE_BROADCAST, EDGE_REPLY, EDGE_RELAY])
-		.range(d3.scale.category10().range());
-
-	var format_time = d3.time.format("%Y-%m-%d");
-
-	var force = d3.layout.force()
-		.charge(-150)
-		.linkDistance(500)
-		.size([width, height]);
-
-	var svg = d3.select("body").append("svg")
-		.attr("width", width)
-		.attr("height", height);
-
-	svg.append("svg:defs")
-		.append("svg:marker")
-		.attr("id", "triangle")
-		.attr("viewBox", "0 -5 10 10")
-		.attr("refX", 15)
-		.attr("refY", -1.5)
-		.attr("markerWidth", 3)
-		.attr("markerHeight", 3)
-		.attr("orient", "auto")
-		.append("svg:path")
-		.attr("d", "M0,-5L10,0L0,5");
-
-	d3.json("data/id2interaction.json", function(error, id2interactions) {
-		d3.json("data/id2people.json", function(error, id2people) {
-			d3.json(data_path, function(error, graphs) {
-				var tip = d3.tip()
-					.attr('class', 'd3-tip')
-				// .offset([100, 20])
-					.html(function(n) {
-						var i = id2interactions[n['message_id']];
-						i['date'] = format_time(new Date(i['datetime']*1000));
-						i['sender'] = id2people[i['sender_id']]['email'].replace("@enron.com", "");
-						i['recipients'] = _.map(i['recipient_ids'], function(k){
-							return id2people[k]['email'].replace("@enron.com", "");
-						}).join("    ");
-						console.log('iteraction:', i);
-						return dict2html(i, ['subject', 'body', 'sender', 'recipients', 'date', 'message_id']);
-					});
-				
-				svg.call(tip);
-
-				var graph = graphs[kth];
-
-				if (error) throw error;
-
-				force
-					.nodes(graph.nodes)
-					.links(graph.edges)
-					.start();
-
-				var link = svg.selectAll(".link")
-					.data(graph.edges)
-  					.enter().append("line")
-					.attr("class", "link")
-					.attr("marker-end", "url(#triangle)")
-					.attr("stroke", function(d){
-						var s = d['source'], t = d['target'];
-						if(s["sender_id"] == t["sender_id"]){
-							return palette(EDGE_BROADCAST); // broadcast
-						}
-						else if(_.intersection(s["recipient_ids"], [t["sender_id"]]) && 
-								_.intersection(t["recipient_ids"], [s["sender_id"]])){
-							return palette(EDGE_REPLY); // reply
-						}
-						else if(_.intersection(s["recipient_ids"], [t["sender_id"]]) && 
-								!_.intersection(t["recipient_ids"], [s["sender_id"]])){
-							return palette(EDGE_RELAY); // relay
-						}
-					})
-					.attr("stroke-width", function(d){
-						if(d['event']){
-							return 4;
-						}else{
-							return 1;
-						}
-					})
-					.attr("opacity", function(d){
-						if(d['event']){
-							return 1;
-						}else{
-							return 0.5;
-						}
-					});
-
-				function mouseover_wrapper(d){					
-					tip.show(d);
-					link.style('stroke-width', function(l) {
-						if (d === l.source)
-							return 2;
-						else
-							return 1;
-					});
-				}
-				function mouseout_wrapper(d){					
-					tip.hide(d);
-					link.style('stroke-width', 1);
-				}
-				var node = svg.selectAll(".node")
-					.data(graph.nodes)
-					.enter().append("circle")
-					.attr("class", "node")
-					.attr("r", 5)
-				// .style("fill", palette(0))
-					.style("fill", function(d){
-						if(d['event']){
-							return palette(0);
-						}else{
-							return '#eee';
-						}
-					})
-					.call(force.drag)			
-					// .on('mouseover', tip.show)
-					// .on('mouseout', tip.hide)
-					.on('mouseover', mouseover_wrapper)
-					.on('mouseout',  mouseout_wrapper)
-
-
-				force.on("tick", function() {
-					link.attr("x1", function(d) { return d.source.x; })
-						.attr("y1", function(d) { return d.source.y; })
-						.attr("x2", function(d) { return d.target.x; })
-						.attr("y2", function(d) { return d.target.y; });
-
-					node.attr("cx", function(d) { return d.x; })
-						.attr("cy", function(d) { return d.y; });
-				});
-			});
-		});
-	});
-}

html/meta_graph_display.html

@@ -33,6 +33,7 @@
 <body>
   <div>
 	<select id="dataset">
+	  <option value="twitter">Twitter</option>
 	  <option value="enron">Enron</option>
 	  <option value="islamic">Islamic forum</option>
 	  <option value="sklearn">Sklearn(Github repository)</option>

html/timeline.html

@@ -13,6 +13,7 @@
   <div>
 	Dataset:
 	<select id="dataset">
+	  <option value="twitter">Twitter</option>
 	  <option value="enron">Enron</option>
 	  <option value="islamic">Islamic forum</option>
 	  <option value="sklearn">Sklearn(Github repository)</option>

meta_graph_stat.py

@@ -178,8 +178,9 @@ def populate_user_info(counter):
                 return Counter(data)
 
             result['sender_count'] = Counter(
-                [peopleid2info[
-                    id2interaction[self.g.node[n]['message_id']]['sender_id']]
+                [peopleid2info.get(
+                    id2interaction[self.g.node[n]['message_id']]['sender_id'],
+                    'unknown')
                  for n in self.g.nodes()]
             )
 

util.py

@@ -3,6 +3,7 @@
 import math
 import gensim
 from datetime import datetime
+from collections import defaultdict
 
 
 def load_items_by_line(path):
@@ -21,14 +22,17 @@ def load_id2obj_dict(path, id_key):
         interactions = json.load(open(path))
     except ValueError:
         interactions = load_json_by_line(path)
-    return {i[id_key]: i
-            for i in interactions}
+    d = defaultdict(lambda: {'id': 'unknown', 'name': 'unknown'})
+    for i in interactions:
+        d[i[id_key]] = i
+    return d
 
 
 def get_datetime(obj):
     if isinstance(obj, datetime):
         return obj
-    elif (isinstance(obj, float) or isinstance(obj, int)) and not math.isnan(obj):
+    elif (isinstance(obj, float) or
+          isinstance(obj, int)) and not math.isnan(obj):
         return datetime.fromtimestamp(obj)
     elif isinstance(obj, long):
         return datetime.fromtimestamp(obj / 1000)

viz_util.py

@@ -5,7 +5,7 @@ def to_d3_graph(g):
     data = {'nodes': [], 'edges': []}
     for n in g.nodes_iter():
         node = g.node[n]
-        for f in ('topics', 'bow'):
+        for f in ('topics', 'bow', 'hashtag_bow'):
             if f in node:
                 del node[f]