PoC‖GTFO
Twitter
LinkedIn
GitHub
XTermI’ve now been using Twitter for about six months. While Twitter’s minimalism is no doubt responsible for much of its success, I often pine for some additional social networking features. High up on that list is a simple way of representing my closest neighbors—perhaps through a visualization—without having to manually navigate individual users’ following/followers pages. A well designed representation could be useful in a number of ways:
- It could expose previously unknown mutual relationships (i.e., “Wow, I didn’t know X and Y knew each other!);
- It could reveal mutual acquaintances whom one did not know were on Twitter; and
- Metrics on the social network could be aggregated (e.g., degrees of separation).
The code for TwitterGraph follows at the end of this post. The code depends on the simplejson module and also imagemagick. It uses the Twitter API to construct the network graph. You don’t need to have a Twitter account for this to work; it doesn’t require authentication. Each IP is, however, limited to 100 API calls per hour, unless your IP has been whitelisted. My script takes this into account. Each Twitter user requires three API to download their information, so one can load about 33 users per hour before reaching the rate limit. TwitterGraph saves its data, so successive calls will continue off where it previously left. Finally, TwitterGraph also calculates the PageRank algorithm).
Usage: paste the code below into TwitterGraph.py and run the following:
$ chmod 755 ./TwitterGraph.py $ ./TwitterGraph.py You have 100 API calls remaining this hour; how many would you like to use now? 80 What is the twitter username for which you'd like to build a graph? ESultanik Building the graph for ESultanik (output will be ESultanik.dot)... . . . $ dot -Tps ESultanik.dot -o ESultanik.ps && epstopdf ESultanik.ps && acroread ESultanik.pdf $ dot -Tsvgz ESultanik.dot -o ESultanik.svgz
There are also (unnecessary) command line options, the usage for which should be evident from the sourcecode.
#!/usr/bin/python
import simplejson
import urllib2
import urllib
import getopt, sys
import re
import os
class TwitterError(Exception):
@property
def message(self):
return self.args[0]
def CheckForTwitterError(data):
if 'error' in data:
raise TwitterError(data['error'])
def fetch_url(url):
opener = urllib2.build_opener()
url_data = opener.open(url).read()
opener.close()
return url_data
def remaining_api_hits():
json = fetch_url("http://twitter.com/account/rate_limit_status.json")
data = simplejson.loads(json)
CheckForTwitterError(data)
return data['remaining_hits']
def get_user_info(id):
global is_username
global calls
json = None
calls += 1
if is_username:
json = fetch_url("http://twitter.com/users/show.json?screen_name=" + str(id))
else:
json = fetch_url("http://twitter.com/users/show.json?user_id=" + str(id))
data = simplejson.loads(json)
CheckForTwitterError(data)
return data
def get_friends(id):
global calls
calls += 1
json = fetch_url("http://twitter.com/friends/ids.json?user_id=" + str(id))
data = simplejson.loads(json)
CheckForTwitterError(data)
return data
def get_followers(id):
global calls
calls += 1
json = fetch_url("http://twitter.com/followers/ids.json?user_id=" + str(id))
data = simplejson.loads(json)
CheckForTwitterError(data)
return data
last_status_msg = ""
def update_status(message):
global last_status_msg
# clear the last message
sys.stdout.write("\r")
p = re.compile(r"[^\s]")
sys.stdout.write(p.sub(' ', last_status_msg))
sys.stdout.write("\r")
sys.stdout.write(message)
last_status_msg = message
sys.stdout.flush()
def clear_status():
last_status_msg = ""
def save_state():
global history
global user_info
global friends
global followers
global queue
global username
data = simplejson.dumps([history, user_info, friends, followers, queue])
bakfile = open(username + ".json", "w")
bakfile.write(data)
bakfile.close()
def build_adjacency():
global friends
idxes = {}
adj = []
idx = 0
for user in friends:
idxes[user] = idx
idx += 1
adj.append([0]*len(friends))
for user in friends:
if len(friends[user]) <= 0:
continue
amount_to_give = 1.0 / len(friends[user])
for f in friends[user]:
if str(f) in idxes:
adj[idxes[user]][idxes[str(f)]] = amount_to_give
return [idxes, adj]
try:
opts, args = getopt.getopt(sys.argv[1:], "hu:c:r", ["help", "user=", "calls=", "resume"])
except getopt.GetoptError, err:
print err
#usage()
sys.exit(2)
max_calls = -1
username = ""
load_prev = None
for o, a in opts:
if o in ("-h", "--help"):
#usage()
sys.exit()
elif o in ("-u", "--user"):
username = a
elif o in ("-c", "--calls"):
max_calls = int(a)
elif o in ("-r", "--resume"):
load_prev = True
else:
assert False, "unhandled option"
if max_calls != 0:
# First, let's find out how many API calls we have left before we are rate limited:
update_status("Contacting Twitter to see how many API calls are left on your account...")
max_hits = remaining_api_hits()
if max_calls < 0 or max_hits < max_calls:
update_status("You have " + str(max_hits) + " API calls remaining this hour; how many would you like to use now? ")
max_calls = int(raw_input())
clear_status()
if max_calls > max_hits:
max_calls = max_hits
if username == "":
print "What is the twitter username for which you'd like to build a graph? ",
username = re.compile(r"\n").sub("", raw_input())
update_status("Trying to open " + username + ".dot for output...")
dotfile = open(username + ".dot", "w")
update_status("")
clear_status()
print "Building the graph for " + username + " (output will be " + username + ".dot)..."
is_username = True
history = {}
queue = [username]
calls = 0
user_info = {}
friends = {}
followers = {}
# Let's see if there's any partial data...
if os.path.isfile(username + ".json"):
print "It appears as if you have some partial data for this user."
resume = ""
if not load_prev:
print "Do you want to start off from where you last finished? (y/n) ",
resume = re.compile(r"\n").sub("", raw_input())
if load_prev == True or resume == "y" or resume == "Y" or resume == "yes" or resume == "Yes" or resume == "YES":
is_username = False
bakfile = open(username + ".json", "r")
[history, user_info, friends, followers, queue] = simplejson.loads(bakfile.read())
print str(len(friends)) + " friends!"
bakfile.close()
print "Loaded " + str(len(history)) + " previous Twitterers!"
print "The current queue size is " + str(len(queue)) + "."
else:
print "You are about to overwrite the partial data; are you sure? (y/n) ",
resume = re.compile(r"\n").sub("", raw_input())
if not (resume == "y" or resume == "Y" or resume == "yes" or resume == "Yes" or resume == "YES"):
exit
while len(queue) > 0 and calls + 3 <= max_calls:
next_user = queue.pop(0)
# Let's just double-check that we haven't already processed this user!
if str(next_user) in history:
continue
update_status(str(next_user) + "\t(? Followers,\t? Following)\tQueue Size: " + str(len(queue)))
if next_user in user_info:
info = user_info[next_user]
else:
try:
info = get_user_info(next_user)
except urllib2.HTTPError:
update_status("It appears as if user " + str(next_user) + "'s account has been suspended!")
print ""
clear_status()
continue
uid = next_user
if is_username:
uid = info['id']
history[uid] = True
is_username = False
user_info[uid] = info
update_status(info['screen_name'] + "\t(? Followers,\t? Following)\tQueue Size: " + str(len(queue)))
followers[uid] = get_followers(uid)
for i in followers[uid]:
if str(i) not in history:
history[i] = True
queue.append(i)
update_status(info['screen_name'] + "\t(" + str(len(followers[uid])) + " Followers,\t? Following)\tQueue Size: " + str(len(queue)))
friends[uid] = get_friends(uid)
for i in friends[uid]:
if str(i) not in history:
history[i] = True
queue.append(i)
update_status(info['screen_name'] + "\t(" + str(len(followers[uid])) + " Followers,\t" + str(len(friends[uid])) + " Following)")
clear_status()
sys.stdout.write("\n")
sys.stdout.flush()
save_state()
# Get some extra user info if we have any API calls remaining
# Find someone in the history for whom we haven't downloaded user info
for user in history:
if calls >= max_calls:
break
if not user in user_info:
try:
user_info[user] = get_user_info(user)
except urllib2.HTTPError:
# This almost always means the user's account has been disabled!
continue
if calls > 0:
save_state()
# Now download any user profile pictures that we might be missing...
update_status("Downloading missing user profile pictures...")
if not os.path.isdir(username + ".images"):
os.mkdir(username + ".images")
user_image_raw = {}
for u in friends:
_, _, filetype = user_info[u]['profile_image_url'].rpartition(".")
filename = username + ".images/" + str(u) + "." + filetype
user_image_raw[u] = filename
if not os.path.isfile(filename):
# we need to download the file!
update_status("Downloading missing user profile picture for " + user_info[u]['screen_name'] + "...")
urllib.urlretrieve(user_info[u]['profile_image_url'], filename)
update_status("Profile pictures are up to date!")
print ""
clear_status()
# Now scale the profile pictures
update_status("Scaling profile pictures...")
user_image = {}
for u in friends:
_, _, filetype = user_info[u]['profile_image_url'].rpartition(".")
filename = username + ".images/" + str(u) + ".scaled." + filetype
user_image[u] = filename
if not os.path.isfile(filename):
# we need to scale the image!
update_status("Scaling profile picture for " + user_info[u]['screen_name'] + "...")
os.system("convert -resize 48x48 " + user_image_raw[u] + " " + user_image[u])
update_status("Profile pictures are all scaled!")
print ""
clear_status()
update_status("Building the adjacency matrix...")
[idxes, adj] = build_adjacency()
print ""
clear_status()
update_status("Calculating the stationary distribution...")
iterations = 500
damping_factor = 0.25
st = [1.0]*len(friends)
last_percent = -1
for i in range(iterations):
users = 0
for u in friends:
users += 1
percent = round(float(i * len(friends) + users) / float(iterations * len(friends)) * 100.0, 1)
if percent > last_percent:
last_percent = percent
update_status("Calculating the stationary distribution... " + str(percent) + "%")
idx = idxes[str(u)]
given_away = 0.0
give_away = st[idx] * (1.0 - damping_factor)
if give_away <= 0.0:
continue
for f in friends[u]:
if str(f) not in friends:
continue
fidx = idxes[str(f)]
ga = adj[idx][fidx] * give_away
given_away += ga
st[fidx] += ga
st[idx] -= given_away
print ""
clear_status()
# Now calculate the ranks of the users
deco = [ (st[idxes[u]], i, u) for i, u in enumerate(friends.keys()) ]
deco.sort()
deco.reverse()
rank = {}
last_st = None
last_rank = 1
for st, _, u in deco:
if last_st == None:
rank[u] = 1
elif st == last_st:
rank[u] = last_rank
else:
rank[u] = last_rank + 1
last_rank = rank[u]
last_st = st
print user_info[u]['screen_name'] + "\t" + str(rank[u])
update_status("Generating the .dot file...")
# Now generate the .dot file
dotfile.write("digraph twitter {\n")
dotfile.write(" /* A TwitterGraph automatically generated by Evan Sultanik's Python script! */\n")
dotfile.write(" /* http://www.sultanik.com/ */\n")
for user in friends:
dotfile.write(" n" + str(user) + " [label=< | " + user_info[user]['name'])
if not (user_info[user]['name'] == user_info[user]['screen_name']):
dotfile.write(" (" + user_info[user]['screen_name'] + ")") dotfile.write(" |
| Rank: " + str(rank[user]) + " |
Atom Feed