Vizualizing Twitter

I’ve now been using Twitter for about six months. While Twitter’s minimalism is no doubt responsible for much of its success, I often pine for some additional social networking features. High up on that list is a simple way of representing my closest neighbors—perhaps through a visualization—without having to manually navigate individual users’ following/followers pages. A well designed representation could be useful in a number of ways:

It could expose previously unknown mutual relationships (i.e., “Wow, I didn’t know X and Y knew each other!);
It could reveal mutual acquaintances whom one did not know were on Twitter; and
Metrics on the social network could be aggregated (e.g., degrees of separation).

This afternoon I spent an hour or so hacking together a Python script, which I have dubbed TwitterGraph, to accomplish this. Here is an example of the ~100 people nearest to me in the network:

The code for TwitterGraph follows at the end of this post. The code depends on the simplejson module and also imagemagick. It uses the Twitter API to construct the network graph. You don’t need to have a Twitter account for this to work; it doesn’t require authentication. Each IP is, however, limited to 100 API calls per hour, unless your IP has been whitelisted. My script takes this into account. Each Twitter user requires three API to download their information, so one can load about 33 users per hour before reaching the rate limit. TwitterGraph saves its data, so successive calls will continue off where it previously left. Finally, TwitterGraph also calculates the PageRank algorithm).

Usage: paste the code below into TwitterGraph.py and run the following:

$ chmod 755 ./TwitterGraph.py
$ ./TwitterGraph.py
You have 100 API calls remaining this hour; how many would you like to use now? 80
What is the twitter username for which you'd like to build a graph? ESultanik
Building the graph for ESultanik (output will be ESultanik.dot)...
.
.
.
$ dot -Tps ESultanik.dot -o ESultanik.ps && epstopdf ESultanik.ps && acroread ESultanik.pdf
$ dot -Tsvgz ESultanik.dot -o ESultanik.svgz

There are also (unnecessary) command line options, the usage for which should be evident from the sourcecode.

#!/usr/bin/python

import simplejson
import urllib2
import urllib
import getopt, sys
import re
import os

class TwitterError(Exception):
  @property
  def message(self):
    return self.args[0]

def CheckForTwitterError(data):
    if 'error' in data:
      raise TwitterError(data['error'])

def fetch_url(url):
    opener = urllib2.build_opener()
    url_data = opener.open(url).read()
    opener.close()
    return url_data

def remaining_api_hits():
    json = fetch_url("http://twitter.com/account/rate_limit_status.json")
    data = simplejson.loads(json)
    CheckForTwitterError(data)
    return data['remaining_hits']

def get_user_info(id):
    global is_username
    global calls
    json = None
    calls += 1
    if is_username:
        json = fetch_url("http://twitter.com/users/show.json?screen_name=" + str(id))
    else:
        json = fetch_url("http://twitter.com/users/show.json?user_id=" + str(id))
    data = simplejson.loads(json)
    CheckForTwitterError(data)
    return data

def get_friends(id):
    global calls
    calls += 1
    json = fetch_url("http://twitter.com/friends/ids.json?user_id=" + str(id))
    data = simplejson.loads(json)
    CheckForTwitterError(data)
    return data

def get_followers(id):
    global calls
    calls += 1
    json = fetch_url("http://twitter.com/followers/ids.json?user_id=" + str(id))
    data = simplejson.loads(json)
    CheckForTwitterError(data)
    return data

last_status_msg = ""
def update_status(message):
    global last_status_msg
    # clear the last message
    sys.stdout.write("\r")
    p = re.compile(r"[^\s]")
    sys.stdout.write(p.sub(' ', last_status_msg))
    sys.stdout.write("\r")
    sys.stdout.write(message)
    last_status_msg = message
    sys.stdout.flush()

def clear_status():
    last_status_msg = ""

def save_state():
    global history
    global user_info
    global friends
    global followers
    global queue
    global username
    data = simplejson.dumps([history, user_info, friends, followers, queue])
    bakfile = open(username + ".json", "w")
    bakfile.write(data)
    bakfile.close()

def build_adjacency():
    global friends
    idxes = {}
    adj = []
    idx = 0
    for user in friends:
        idxes[user] = idx
        idx += 1
        adj.append([0]*len(friends))
    for user in friends:
        if len(friends[user]) <= 0:
            continue
        amount_to_give = 1.0 / len(friends[user])
        for f in friends[user]:
            if str(f) in idxes:
                adj[idxes[user]][idxes[str(f)]] = amount_to_give
    return [idxes, adj]

try:
    opts, args = getopt.getopt(sys.argv[1:], "hu:c:r", ["help", "user=", "calls=", "resume"])
except getopt.GetoptError, err:
    print err
    #usage()
    sys.exit(2)

max_calls = -1
username = ""
load_prev = None

for o, a in opts:
    if o in ("-h", "--help"):
        #usage()
        sys.exit()
    elif o in ("-u", "--user"):
        username = a
    elif o in ("-c", "--calls"):
        max_calls = int(a)
    elif o in ("-r", "--resume"):
        load_prev = True
    else:
        assert False, "unhandled option"

if max_calls != 0:
    # First, let's find out how many API calls we have left before we are rate limited:
    update_status("Contacting Twitter to see how many API calls are left on your account...")
    max_hits = remaining_api_hits()
    if max_calls < 0 or max_hits < max_calls:
        update_status("You have " + str(max_hits) + " API calls remaining this hour; how many would you like to use now? ")
        max_calls = int(raw_input())
        clear_status()
        if max_calls > max_hits:
            max_calls = max_hits
if username == "":
    print "What is the twitter username for which you'd like to build a graph? ",
    username = re.compile(r"\n").sub("", raw_input())

update_status("Trying to open " + username + ".dot for output...")
dotfile = open(username + ".dot", "w")
update_status("")
clear_status()
print "Building the graph for " + username + " (output will be " + username + ".dot)..."

is_username = True
history = {}
queue = [username]
calls = 0
user_info = {}
friends = {}
followers = {}

# Let's see if there's any partial data...
if os.path.isfile(username + ".json"):
    print "It appears as if you have some partial data for this user."
    resume = ""
    if not load_prev:
        print "Do you want to start off from where you last finished? (y/n) ",
        resume = re.compile(r"\n").sub("", raw_input())
    if load_prev == True or resume == "y" or resume == "Y" or resume == "yes" or resume == "Yes" or resume == "YES":
        is_username = False
        bakfile = open(username + ".json", "r")
        [history, user_info, friends, followers, queue] = simplejson.loads(bakfile.read())
        print str(len(friends)) + " friends!"
        bakfile.close()
        print "Loaded " + str(len(history)) + " previous Twitterers!"
        print "The current queue size is " + str(len(queue)) + "."
    else:
        print "You are about to overwrite the partial data; are you sure? (y/n) ",
        resume = re.compile(r"\n").sub("", raw_input())
        if not (resume == "y" or resume == "Y" or resume == "yes" or resume == "Yes" or resume == "YES"):
            exit

while len(queue) > 0 and calls + 3 <= max_calls:
    next_user = queue.pop(0)
    # Let's just double-check that we haven't already processed this user!
    if str(next_user) in history:
        continue
    update_status(str(next_user) + "\t(? Followers,\t? Following)\tQueue Size: " + str(len(queue)))
    if next_user in user_info:
        info = user_info[next_user]
    else:
        try:
            info = get_user_info(next_user)
        except urllib2.HTTPError:
            update_status("It appears as if user " + str(next_user) + "'s account has been suspended!")
            print ""
            clear_status()
            continue
    uid = next_user
    if is_username:
        uid = info['id']
        history[uid] = True
        is_username = False
    user_info[uid] = info
    update_status(info['screen_name'] + "\t(? Followers,\t? Following)\tQueue Size: " + str(len(queue)))
    followers[uid] = get_followers(uid)
    for i in followers[uid]:
        if str(i) not in history:
            history[i] = True
            queue.append(i)
    update_status(info['screen_name'] + "\t(" + str(len(followers[uid])) + " Followers,\t? Following)\tQueue Size: " + str(len(queue)))
    friends[uid] = get_friends(uid)
    for i in friends[uid]:
        if str(i) not in history:
            history[i] = True
            queue.append(i)
    update_status(info['screen_name'] + "\t(" + str(len(followers[uid])) + " Followers,\t" + str(len(friends[uid])) + " Following)")
    clear_status()
    sys.stdout.write("\n")
    sys.stdout.flush()
    save_state()

# Get some extra user info if we have any API calls remaining
# Find someone in the history for whom we haven't downloaded user info
for user in history:
    if calls >= max_calls:
        break
    if not user in user_info:
        try:
            user_info[user] = get_user_info(user)
        except urllib2.HTTPError:
            # This almost always means the user's account has been disabled!
            continue

if calls > 0:
    save_state()

# Now download any user profile pictures that we might be missing...
update_status("Downloading missing user profile pictures...")
if not os.path.isdir(username + ".images"):
    os.mkdir(username + ".images")
user_image_raw = {}
for u in friends:
    _, _, filetype = user_info[u]['profile_image_url'].rpartition(".")
    filename = username + ".images/" + str(u) + "." + filetype
    user_image_raw[u] = filename
    if not os.path.isfile(filename):
        # we need to download the file!
        update_status("Downloading missing user profile picture for " + user_info[u]['screen_name'] + "...")
        urllib.urlretrieve(user_info[u]['profile_image_url'], filename)
update_status("Profile pictures are up to date!")
print ""
clear_status()

# Now scale the profile pictures
update_status("Scaling profile pictures...")
user_image = {}
for u in friends:
    _, _, filetype = user_info[u]['profile_image_url'].rpartition(".")
    filename = username + ".images/" + str(u) + ".scaled." + filetype
    user_image[u] = filename
    if not os.path.isfile(filename):
        # we need to scale the image!
        update_status("Scaling profile picture for " + user_info[u]['screen_name'] + "...")
        os.system("convert -resize 48x48 " + user_image_raw[u] + " " + user_image[u])
update_status("Profile pictures are all scaled!")
print ""
clear_status()

update_status("Building the adjacency matrix...")
[idxes, adj] = build_adjacency()
print ""
clear_status()
update_status("Calculating the stationary distribution...")
iterations = 500
damping_factor = 0.25
st = [1.0]*len(friends)
last_percent = -1
for i in range(iterations):
    users = 0
    for u in friends:
        users += 1
        percent = round(float(i * len(friends) + users) / float(iterations * len(friends)) * 100.0, 1)
        if percent > last_percent:
            last_percent = percent
            update_status("Calculating the stationary distribution... " + str(percent) + "%")
        idx = idxes[str(u)]
        given_away = 0.0
        give_away = st[idx] * (1.0 - damping_factor)
        if give_away <= 0.0:
            continue
        for f in friends[u]:
            if str(f) not in friends:
                continue
            fidx = idxes[str(f)]
            ga = adj[idx][fidx] * give_away
            given_away += ga
            st[fidx] += ga
        st[idx] -= given_away
print ""
clear_status()
# Now calculate the ranks of the users
deco = [ (st[idxes[u]], i, u) for i, u in enumerate(friends.keys()) ]
deco.sort()
deco.reverse()
rank = {}
last_st = None
last_rank = 1
for st, _, u in deco:
    if last_st == None:
        rank[u] = 1
    elif st == last_st:
        rank[u] = last_rank
    else:
        rank[u] = last_rank + 1
    last_rank = rank[u]
    last_st = st
    print user_info[u]['screen_name'] + "\t" + str(rank[u])

update_status("Generating the .dot file...")

# Now generate the .dot file
dotfile.write("digraph twitter {\n")
dotfile.write("  /* A TwitterGraph automatically generated by Evan Sultanik's Python script! */\n")
dotfile.write("  /* http://www.sultanik.com/                                                 */\n")
for user in friends:
    dotfile.write("  n" + str(user) + " [label=< ")
    dotfile.write("")
    dotfile.write("" + user_info[user]['name'])
    if not (user_info[user]['name'] == user_info[user]['screen_name']):
        dotfile.write("
(" + user_info[user]['screen_name'] + ")")
    dotfile.write("
Rank: " + str(rank[user]) + "
 >");
    if user_info[user]['screen_name'] == username:
        dotfile.write(" color=\"green\" shape=\"doubleoctagon\"")
    dotfile.write("];\n")
dotfile.write("\n")
for user in friends:
    for f in friends[user]:
        if str(f) in friends:
            dotfile.write("  n" + str(user) + " -> " + " n" + str(f) + ";\n")
dotfile.write("}\n")

dotfile.close()

print ""
clear_status()

← Older Post

Newer Post →

Vizualizing Twitter