Facebook Government Requests Per User

import pandas as pd
import matplotlib as mpl
import numpy as np
import requests
import lxml.html
import json
from StringIO import StringIO

from IPython.display import HTML
from markdown import markdown
md = lambda x: HTML(markdown(x))

import mplstyle, mplstyle.styles.simple
mplstyle.set(mplstyle.styles.simple)
mplstyle.set({
    "figure.figsize": (10, 15),
})
base_styles = mplstyle.get()

FB_GOV_URL = "https://www.facebook.com/about/government_requests/download.php"
fb_gov_csv = requests.get(FB_GOV_URL).text
fb_gov = pd.read_csv(StringIO(fb_gov_csv), 
    names=["country", "requests", "users_requested", "percent_complied"], 
    header=False).set_index("country")

def parse_fb_counts(s):
    commaless = str(s).replace(",", "")
    split = commaless.split(" - ")
    mean = np.mean(map(float, split))
    return mean

fb_gov["requests"] = fb_gov["requests"].apply(parse_fb_counts)
fb_gov["users_requested"] = fb_gov["users_requested"].apply(parse_fb_counts)

fb_gov.sort("users_requested", ascending=False)["users_requested"].head(10).reset_index()

FB_USERS_URL = "http://en.wikipedia.org/wiki/Facebook_statistics"
fb_users_text = requests.get(FB_USERS_URL).text
fb_users_dom = lxml.html.fromstring(fb_users_text)

fb_user_rows = fb_users_dom.cssselect("table.wikitable")[0].cssselect("tr")[1:]

def parse_user_row(row):
    cells = row.cssselect("td")
    return {
        "country": cells[1].cssselect("a")[0].text,
        "fb_users": int(cells[4].text.replace(",", ""))
    }

fb_users = pd.DataFrame(map(parse_user_row, fb_user_rows)).set_index("country")

fb = fb_gov.join(fb_users, how="left")

missing_usage = fb[fb["fb_users"].isnull()]
md("The two datasets match up well; there are only %d countries in the government-requests dataset missing from the usage dataset:\n\n %s" % 
(len(missing_usage), "\n".join("- %s" % x for x in missing_usage.index)))

def plot_fb_scatter():
    mplstyle.set({ "figure.figsize": (8, 8) })
    ax = mpl.pyplot.scatter(fb["fb_users"].apply(log10), fb["users_requested"].apply(log10), 
        s=20, 
        alpha=0.7).axes
    ax.set_ylim(0, ax.get_ylim()[1] * 1.1)
    ax.set_title("Facebook Users vs. Accounts Requested, by Country\n")
    ax.set_xlabel("\nLog10 of Facebook Users, End of 2012")
    ax.set_ylabel("Log10 of Accounts Requested, First Half of 2013\n")
    mplstyle.reset(base_styles)

plot_fb_scatter()

pearsons_r = fb["fb_users"].apply(log10).corr(fb["users_requested"].apply(log10))
md("""
Indeed, it appears that countries with more Facebook users tend to request data on a greater number of user accounts.
(The *Pearson's r* is %.2f, which suggests a fairly strong, positive correlation.)
""" % round(pearsons_r, 2))

fb["per_m_users"] = fb["users_requested"] * 1e6 / fb["fb_users"]

def plot_request_rate():
    fb_sort_rate = fb[fb["fb_users"].notnull() * (fb["users_requested"] > 10)].sort("per_m_users")
    ax = fb_sort_rate["per_m_users"].plot(kind="barh", color="teal", alpha=0.5)
    ax.grid(axis="y")
    for i in range(len(fb_sort_rate)):
        row = fb_sort_rate.irow(i)
        ax.text(row["per_m_users"], i+0.6, "% 0.1f" % row["per_m_users"], va="center")
    ax.set_xlim(0, ax.get_xlim()[1] * 1.1)
    ax.set_title("Facebook Accounts Requested Per Million Users\nDuring the First Six Months of 2013, by Country\n")
    ax.set_ylabel("")
    ax.set_xlabel("\nAccounts Requested Per Million Users")

plot_request_rate()

commafy = lambda x: "{:,}".format(int(round(x)))

malta = fb.ix["Malta"]
next_highest = fb["per_m_users"].order(ascending=False).iget(1)
md("""
__Malta requested data on %d Facebook accounts per million users, more than %d times the next-highest rate.__
Flipping the denominator, that's 1 in every %s users.
""" % (
    round(malta["per_m_users"]),
    int(malta["per_m_users"] / next_highest),
    commafy(round(malta["fb_users"] / malta["users_requested"])))
)

cols = ["country", "users_requested", "fb_users", "per_m_users"]
fb[fb["fb_users"].notnull()].sort("per_m_users", ascending=False).reset_index()[cols]