2_tech_nonprofit_irs_comp_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://s3.amazonaws.com/irs-form-990/201120429349301812_public.xml\n",
      "The Center to Promote Healthcare Access Inc 11\n",
      "https://s3.amazonaws.com/irs-form-990/201102279349304550_public.xml\n",
      "CENTER FOR STUDENT OPPORTUNITY INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201132989349300003_public.xml\n",
      "CLASSROOM INC 15\n",
      "https://s3.amazonaws.com/irs-form-990/201103139349300945_public.xml\n",
      "CODE FOR AMERICA LABS INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201143199349310864_public.xml\n",
      "D-REV DESIGN FOR THE OTHER 90 8\n",
      "https://s3.amazonaws.com/irs-form-990/201122989349300912_public.xml\n",
      "FAMILIES EMPOWERED 2\n",
      "https://s3.amazonaws.com/irs-form-990/201100329349200010_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE FREECYCLE NETWORK 3\n",
      "https://s3.amazonaws.com/irs-form-990/201133189349303268_public.xml\n",
      "HARMONY INSTITUTE INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201133199349101553_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HopeLab Foundation Inc 9\n",
      "https://s3.amazonaws.com/irs-form-990/201113199349301711_public.xml\n",
      "IN OUR BACKYARDS INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201123199349309522_public.xml\n",
      "IMAGING THE WORLD CORPORATION 15\n",
      "https://s3.amazonaws.com/irs-form-990/201143119349100554_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE KIVA FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201103189349307545_public.xml\n",
      "LITERACY BRIDGE 4\n",
      "https://s3.amazonaws.com/irs-form-990/201113199349309251_public.xml\n",
      "ONE LAPTOP PER CHILD ASSOCIATION INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201131959349300623_public.xml\n",
      "OPEN MEDIA FOUNDATION 11\n",
      "https://s3.amazonaws.com/irs-form-990/201112249349300801_public.xml\n",
      "PARTICIPATORY CULTURE FOUNDATION INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201103199349309475_public.xml\n",
      "PARTICIPATORY POLITICS FOUNDATION 4\n",
      "https://s3.amazonaws.com/irs-form-990/201123199349305262_public.xml\n",
      "SAMASOURCE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201121349349300517_public.xml\n",
      "SkyTruth 7\n",
      "https://s3.amazonaws.com/irs-form-990/201121369349305607_public.xml\n",
      "TECHSOUP GLOBAL 21\n",
      "https://s3.amazonaws.com/irs-form-990/201120469349300637_public.xml\n",
      "Video Volunteers 11\n",
      "https://s3.amazonaws.com/irs-form-990/201130919349300623_public.xml\n",
      "Wikimedia Foundation Inc 14\n",
      "https://s3.amazonaws.com/irs-form-990/201103189349303710_public.xml\n",
      "iCivics Inc 17\n",
      "https://s3.amazonaws.com/irs-form-990/201120469349301227_public.xml\n",
      "iCivics Inc 13\n",
      "https://s3.amazonaws.com/irs-form-990/201200689349301005_public.xml\n",
      "ADOPT-A-CLASSROOM INC 13\n",
      "https://s3.amazonaws.com/irs-form-990/201243159349300019_public.xml\n",
      "Blue Planet Network 9\n",
      "https://s3.amazonaws.com/irs-form-990/201232929349300613_public.xml\n",
      "CLASSROOM INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201202569349300710_public.xml\n",
      "COMMON SENSE MEDIA 33\n",
      "https://s3.amazonaws.com/irs-form-990/201211329349302176_public.xml\n",
      "Do Something Inc 11\n",
      "https://s3.amazonaws.com/irs-form-990/201221329349301047_public.xml\n",
      "Do Something Inc 11\n",
      "https://s3.amazonaws.com/irs-form-990/201202849349300825_public.xml\n",
      "DTWO LTD 3\n",
      "https://s3.amazonaws.com/irs-form-990/201201369349201200_public.xml\n",
      "year formed\n",
      "n employees\n",
      "DEMOCRACY WORKS INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201241359349303439_public.xml\n",
      "DESIGN THAT MATTERS INC 2\n",
      "https://s3.amazonaws.com/irs-form-990/201203199349302890_public.xml\n",
      "DIGITAL GREEN FOUNDATION 5\n",
      "https://s3.amazonaws.com/irs-form-990/201223199349302697_public.xml\n",
      "DIGITAL GREEN FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201202419349300615_public.xml\n",
      "EDNOVO 1\n",
      "https://s3.amazonaws.com/irs-form-990/201213189349301046_public.xml\n",
      "FAMILIES EMPOWERED 3\n",
      "https://s3.amazonaws.com/irs-form-990/201233199349202593_public.xml\n",
      "year formed\n",
      "n employees\n",
      "GLOBAL LIVES PROJECT INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201233079349300323_public.xml\n",
      "Great NonProfits 11\n",
      "https://s3.amazonaws.com/irs-form-990/201213199349101501_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HopeLab Foundation Inc 10\n",
      "https://s3.amazonaws.com/irs-form-990/201233209349308583_public.xml\n",
      "IMAGING THE WORLD CORPORATION 13\n",
      "https://s3.amazonaws.com/irs-form-990/201213219349300501_public.xml\n",
      "InSTEDD 8\n",
      "https://s3.amazonaws.com/irs-form-990/201232549349100103_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE KIVA FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201223209349305537_public.xml\n",
      "LITERACY BRIDGE 4\n",
      "https://s3.amazonaws.com/irs-form-990/201231249349300743_public.xml\n",
      "LITERACY LAB 6\n",
      "https://s3.amazonaws.com/irs-form-990/201212409349300021_public.xml\n",
      "MIND RESEARCH INSTITUTE 35\n",
      "https://s3.amazonaws.com/irs-form-990/201240629349301609_public.xml\n",
      "National Institute on Money in State 10\n",
      "https://s3.amazonaws.com/irs-form-990/201243189349304369_public.xml\n",
      "NEWBORN FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201212209349301091_public.xml\n",
      "NEXLEAF ANALYTICS 5\n",
      "https://s3.amazonaws.com/irs-form-990/201233209349305723_public.xml\n",
      "ONE LAPTOP PER CHILD ASSOCIATION INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201222999349300912_public.xml\n",
      "OPEN MEDIA FOUNDATION 8\n",
      "https://s3.amazonaws.com/irs-form-990/201242149349300724_public.xml\n",
      "PARTICIPATORY CULTURE FOUNDATION INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201203479349300040_public.xml\n",
      "PARTICIPATORY POLITICS FOUNDATION 4\n",
      "https://s3.amazonaws.com/irs-form-990/201223209349303937_public.xml\n",
      "SAMASOURCE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201232769349300703_public.xml\n",
      "Single Stop USA Inc 18\n",
      "https://s3.amazonaws.com/irs-form-990/201221369349305462_public.xml\n",
      "SKYTRUTH 6\n",
      "https://s3.amazonaws.com/irs-form-990/201211369349307016_public.xml\n",
      "TECHSOUP GLOBAL 22\n",
      "https://s3.amazonaws.com/irs-form-990/201233209349307078_public.xml\n",
      "UNIVERSALGIVING 7\n",
      "https://s3.amazonaws.com/irs-form-990/201240869349300634_public.xml\n",
      "Video Volunteers 13\n",
      "https://s3.amazonaws.com/irs-form-990/201241369349302854_public.xml\n",
      "WISHBONEORG 3\n",
      "https://s3.amazonaws.com/irs-form-990/201241309349302274_public.xml\n",
      "Wikimedia Foundation Inc 17\n",
      "https://s3.amazonaws.com/irs-form-990/201242139349300439_public.xml\n",
      "WORLDREADERORG 3\n",
      "https://s3.amazonaws.com/irs-form-990/201223079349301162_public.xml\n",
      "iCivics Inc 16\n",
      "https://s3.amazonaws.com/irs-form-990/201312269349201931_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE 1947 PARTITION ARCHIVE 2\n",
      "https://s3.amazonaws.com/irs-form-990/201311359349305906_public.xml\n",
      "ADOPT-A-CLASSROOM INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201332229349300303_public.xml\n",
      "Blue Planet Network 8\n",
      "https://s3.amazonaws.com/irs-form-990/201341309349301729_public.xml\n",
      "THE CENTER TO PROMOTE HEALTHCARE ACCESS INC 11\n",
      "https://s3.amazonaws.com/irs-form-990/201322279349100507_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CASE COMMONS INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201330429349301363_public.xml\n",
      "CENTER FOR STUDENT OPPORTUNITY INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201330449349300513_public.xml\n",
      "CODE FOR AMERICA LABS INC 8\n",
      "https://s3.amazonaws.com/irs-form-990/201333189349304218_public.xml\n",
      "COMMON SENSE MEDIA 38\n",
      "https://s3.amazonaws.com/irs-form-990/201343179349302349_public.xml\n",
      "D-Rev Design for the Other 90 7\n",
      "https://s3.amazonaws.com/irs-form-990/201331289349301198_public.xml\n",
      "Do Something Inc 13\n",
      "https://s3.amazonaws.com/irs-form-990/201301709349300320_public.xml\n",
      "DEMOCRACY WORKS INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201341709349300114_public.xml\n",
      "DESIGN THAT MATTERS INC 2\n",
      "https://s3.amazonaws.com/irs-form-990/201303049349300500_public.xml\n",
      "EDNOVO 3\n",
      "https://s3.amazonaws.com/irs-form-990/201343189349302609_public.xml\n",
      "FAIR TRADE FUND INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201302559349300700_public.xml\n",
      "FAMILIES EMPOWERED 3\n",
      "https://s3.amazonaws.com/irs-form-990/201321359349305357_public.xml\n",
      "THE FREECYCLE NETWORK 3\n",
      "https://s3.amazonaws.com/irs-form-990/201311289349301131_public.xml\n",
      "THE GET SCHOOLED FOUNDATION 9\n",
      "https://s3.amazonaws.com/irs-form-990/201310159349301326_public.xml\n",
      "GIVE DIRECT INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201322059349300132_public.xml\n",
      "GOOD WORLD SOLUTIONS INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201332269349301263_public.xml\n",
      "Great NonProfits 14\n",
      "https://s3.amazonaws.com/irs-form-990/201313189349306691_public.xml\n",
      "HARMONY INSTITUTE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201330179349300133_public.xml\n",
      "HARMONY INSTITUTE INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201341359349203334_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HOLLABACK INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201333179349100853_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HopeLab Foundation Inc 10\n",
      "https://s3.amazonaws.com/irs-form-990/201343169349303194_public.xml\n",
      "HUMANITARIAN OPENSTREETMAP TEAM UNITED STATES INC 8\n",
      "https://s3.amazonaws.com/irs-form-990/201330329349300703_public.xml\n",
      "IN OUR BACKYARDS INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201303199349302740_public.xml\n",
      "IN OUR BACKYARDS INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201313479349200316_public.xml\n",
      "year formed\n",
      "n employees\n",
      "IMAGING THE WORLD CORPORATION 11\n",
      "https://s3.amazonaws.com/irs-form-990/201302629349300710_public.xml\n",
      "INTERNEWS NETWORK INC 21\n",
      "https://s3.amazonaws.com/irs-form-990/201311309349100016_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE KIVA FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201303179349306290_public.xml\n",
      "KHAN ACADEMY INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201342249349300249_public.xml\n",
      "THE LITERACY LAB 6\n",
      "https://s3.amazonaws.com/irs-form-990/201342249349301364_public.xml\n",
      "year formed\n",
      "The Lunchbox Fund 4\n",
      "https://s3.amazonaws.com/irs-form-990/201341929349300524_public.xml\n",
      "MIND RESEARCH INSTITUTE 26\n",
      "https://s3.amazonaws.com/irs-form-990/201323509349300747_public.xml\n",
      "Moneythink 6\n",
      "https://s3.amazonaws.com/irs-form-990/201300819349300900_public.xml\n",
      "NEW CLASSROOMS INNOVATION PARTNERSINC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201340439349301859_public.xml\n",
      "National Institute on Money in State 10\n",
      "https://s3.amazonaws.com/irs-form-990/201332329349300233_public.xml\n",
      "NEWBORN FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201332359349300708_public.xml\n",
      "NEXLEAF ANALYTICS 7\n",
      "https://s3.amazonaws.com/irs-form-990/201303029349300400_public.xml\n",
      "NPOWERNY INC 26\n",
      "https://s3.amazonaws.com/irs-form-990/201342709349300039_public.xml\n",
      "OPEN MEDIA FOUNDATION 8\n",
      "https://s3.amazonaws.com/irs-form-990/201312249349300731_public.xml\n",
      "PARTICIPATORY CULTURE FOUNDATION INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201303179349303935_public.xml\n",
      "PARTICIPATORY POLITICS FOUNDATION 4\n",
      "https://s3.amazonaws.com/irs-form-990/201341369349200709_public.xml\n",
      "year formed\n",
      "n employees\n",
      "REALLOCATE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201332319349300013_public.xml\n",
      "Single Stop USA Inc 20\n",
      "https://s3.amazonaws.com/irs-form-990/201322269349302302_public.xml\n",
      "SkyTruth 7\n",
      "https://s3.amazonaws.com/irs-form-990/201301349349307710_public.xml\n",
      "TECHSOUP GLOBAL 21\n",
      "https://s3.amazonaws.com/irs-form-990/201310359349300441_public.xml\n",
      "Video Volunteers 13\n",
      "https://s3.amazonaws.com/irs-form-990/201331309349303228_public.xml\n",
      "WISHBONEORG 3\n",
      "https://s3.amazonaws.com/irs-form-990/201321069349300127_public.xml\n",
      "Wikimedia Foundation Inc 18\n",
      "https://s3.amazonaws.com/irs-form-990/201312219349200756_public.xml\n",
      "year formed\n",
      "n employees\n",
      "ZIDISHA INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201342979349300624_public.xml\n",
      "iCivics Inc 17\n",
      "https://s3.amazonaws.com/irs-form-990/201442249349201164_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE 1947 PARTITION ARCHIVE 2\n",
      "https://s3.amazonaws.com/irs-form-990/201400229349300440_public.xml\n",
      "ADOPT-A-CLASSROOM INC 13\n",
      "https://s3.amazonaws.com/irs-form-990/201403189349303940_public.xml\n",
      "ADOPT-A-CLASSROOM INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201402259349303355_public.xml\n",
      "Anjna Patient Education 6\n",
      "https://s3.amazonaws.com/irs-form-990/201421329349300247_public.xml\n",
      "THE CENTER TO PROMOTE HEALTHCARE ACCESS INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201443219349306104_public.xml\n",
      "CODE-TO-LEARN FOUNDATION 5\n",
      "https://s3.amazonaws.com/irs-form-990/201442259349302539_public.xml\n",
      "CODEORG 9\n",
      "https://s3.amazonaws.com/irs-form-990/201413379349300421_public.xml\n",
      "COWORKERORG 5\n",
      "https://s3.amazonaws.com/irs-form-990/201421189349300932_public.xml\n",
      "CANCER COMMONS 7\n",
      "https://s3.amazonaws.com/irs-form-990/201441049349200834_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CAREERVILLAGE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201421029349200507_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CAREERVILLAGE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201413179349100046_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CASE COMMONS INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201420439349301802_public.xml\n",
      "CENTER FOR STUDENT OPPORTUNITY INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201403089349300020_public.xml\n",
      "CLASSROOM INC 19\n",
      "https://s3.amazonaws.com/irs-form-990/201322909349300512_public.xml\n",
      "CLASSROOM INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201400439349301310_public.xml\n",
      "CODE FOR AMERICA LABS INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201422169349300032_public.xml\n",
      "COMMON SENSE MEDIA 38\n",
      "https://s3.amazonaws.com/irs-form-990/201403219349304515_public.xml\n",
      "D-REV DESIGN FOR THE OTHER 90 9\n",
      "https://s3.amazonaws.com/irs-form-990/201442119349300934_public.xml\n",
      "DO SOMETHING INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201403149349301740_public.xml\n",
      "DONORSCHOOSEORG 23\n",
      "https://s3.amazonaws.com/irs-form-990/201443219349310934_public.xml\n",
      "DTWO LTD 4\n",
      "https://s3.amazonaws.com/irs-form-990/201313199349309261_public.xml\n",
      "DTWO LTD 3\n",
      "https://s3.amazonaws.com/irs-form-990/201443169349303409_public.xml\n",
      "DEMOCRACY WORKS INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201430939349300718_public.xml\n",
      "DESIGN THAT MATTERS INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201333199349305408_public.xml\n",
      "DIGITAL GREEN FOUNDATION 7\n",
      "https://s3.amazonaws.com/irs-form-990/201413219349304766_public.xml\n",
      "DIGITAL GREEN FOUNDATION 7\n",
      "https://s3.amazonaws.com/irs-form-990/201443079349300704_public.xml\n",
      "EDNOVO 2\n",
      "https://s3.amazonaws.com/irs-form-990/201442209349301109_public.xml\n",
      "FAIR TRADE FUND INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201423039349301302_public.xml\n",
      "FAMILIES EMPOWERED 3\n",
      "https://s3.amazonaws.com/irs-form-990/201420139349300007_public.xml\n",
      "THE FREECYCLE NETWORK 3\n",
      "https://s3.amazonaws.com/irs-form-990/201441349349302264_public.xml\n",
      "THE GET SCHOOLED FOUNDATION 9\n",
      "https://s3.amazonaws.com/irs-form-990/201410869349300616_public.xml\n",
      "GIVE DIRECT INC 0\n",
      "https://s3.amazonaws.com/irs-form-990/201433189349200323_public.xml\n",
      "year formed\n",
      "n employees\n",
      "GLOBAL LIVES PROJECT INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201422819349300747_public.xml\n",
      "GOOD WORLD SOLUTIONS INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201411989349300021_public.xml\n",
      "Great NonProfits 12\n",
      "https://s3.amazonaws.com/irs-form-990/201303199349308180_public.xml\n",
      "HOPING HEARTS FOUNDATION INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201433169349304393_public.xml\n",
      "HOPING HEARTS FOUNDATION INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201423229349300622_public.xml\n",
      "HARMONY INSTITUTE INC 2\n",
      "https://s3.amazonaws.com/irs-form-990/201430889349300413_public.xml\n",
      "HOLLABACK INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201413189349102001_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HopeLab Foundation Inc 5\n",
      "https://s3.amazonaws.com/irs-form-990/201443219349305954_public.xml\n",
      "IN OUR BACKYARDS INC 19\n",
      "https://s3.amazonaws.com/irs-form-990/201423179349201142_public.xml\n",
      "year formed\n",
      "n employees\n",
      "IMAGING THE WORLD CORP 10\n",
      "https://s3.amazonaws.com/irs-form-990/201403219349303840_public.xml\n",
      "InSTEDD 9\n",
      "https://s3.amazonaws.com/irs-form-990/201333199349310773_public.xml\n",
      "InSTEDD 9\n",
      "https://s3.amazonaws.com/irs-form-990/201433179349305208_public.xml\n",
      "INTERNEWS NETWORK INC 22\n",
      "https://s3.amazonaws.com/irs-form-990/201441259349100734_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE KIVA FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201413189349302501_public.xml\n",
      "KHAN ACADEMY INC 17\n",
      "https://s3.amazonaws.com/irs-form-990/201441329349201729_public.xml\n",
      "year formed\n",
      "n employees\n",
      "LIFEBOX FOUNDATION INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201413189349309466_public.xml\n",
      "Literacy Bridge 6\n",
      "https://s3.amazonaws.com/irs-form-990/201323199349310847_public.xml\n",
      "LITERACY BRIDGE 4\n",
      "https://s3.amazonaws.com/irs-form-990/201412239349301626_public.xml\n",
      "THE LITERACY LAB 7\n",
      "https://s3.amazonaws.com/irs-form-990/201433159349301688_public.xml\n",
      "year formed\n",
      "The Lunchbox Fund 4\n",
      "https://s3.amazonaws.com/irs-form-990/201432359349300303_public.xml\n",
      "MIND RESEARCH INSTITUTE 27\n",
      "https://s3.amazonaws.com/irs-form-990/201333229349300448_public.xml\n",
      "MEDIC MOBILE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201403229349301375_public.xml\n",
      "MEDIC MOBILE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201401349349305055_public.xml\n",
      "NEW CLASSROOMS INNOVATION PARTNERSINC 13\n",
      "https://s3.amazonaws.com/irs-form-990/201343299349300619_public.xml\n",
      "NATIONAL INSTITUTE ON MONEY IN STATE POLITICS 11\n",
      "https://s3.amazonaws.com/irs-form-990/201402259349302940_public.xml\n",
      "NEXLEAF ANALYTICS 7\n",
      "https://s3.amazonaws.com/irs-form-990/201403219349309225_public.xml\n",
      "OCEARCH 10\n",
      "https://s3.amazonaws.com/irs-form-990/201313199349311756_public.xml\n",
      "OCEARCH 11\n",
      "https://s3.amazonaws.com/irs-form-990/201343199349309234_public.xml\n",
      "ONE LAPTOP PER CHILD ASSOCIATION INC 9\n",
      "https://s3.amazonaws.com/irs-form-990/201412259349302466_public.xml\n",
      "OPEN MEDIA FOUNDATION 9\n",
      "https://s3.amazonaws.com/irs-form-990/201421329349300447_public.xml\n",
      "THE OPENGOV FOUNDATION 7\n",
      "https://s3.amazonaws.com/irs-form-990/201413009349300026_public.xml\n",
      "OPERATION ASHA NFP 13\n",
      "https://s3.amazonaws.com/irs-form-990/201403119349301295_public.xml\n",
      "PARTICIPATORY CULTURE FOUNDATION INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201423179349306547_public.xml\n",
      "PARTICIPATORY POLITICS FOUNDATION 4\n",
      "https://s3.amazonaws.com/irs-form-990/201410359349200726_public.xml\n",
      "year formed\n",
      "n employees\n",
      "PULSEPOINT FOUNDATION 6\n",
      "https://s3.amazonaws.com/irs-form-990/201442809349300529_public.xml\n",
      "PULSEPOINT FOUNDATION 6\n",
      "https://s3.amazonaws.com/irs-form-990/201411889349300766_public.xml\n",
      "ReadworksInc 23\n",
      "https://s3.amazonaws.com/irs-form-990/201401069349300205_public.xml\n",
      "REASONING MIND INC 29\n",
      "https://s3.amazonaws.com/irs-form-990/201341489349100414_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE STELLAR FOUNDATION 5\n",
      "https://s3.amazonaws.com/irs-form-990/201401649349100200_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE STELLAR FOUNDATION 5\n",
      "https://s3.amazonaws.com/irs-form-990/201433219349310743_public.xml\n",
      "SAMASOURCE INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201442249349301454_public.xml\n",
      "SCIENCE BUDDIES 7\n",
      "https://s3.amazonaws.com/irs-form-990/201402139349300835_public.xml\n",
      "Single Stop USA Inc 20\n",
      "https://s3.amazonaws.com/irs-form-990/201402759349300605_public.xml\n",
      "SkyTruth 7\n",
      "https://s3.amazonaws.com/irs-form-990/201421359349308847_public.xml\n",
      "TECHSOUP GLOBAL 19\n",
      "https://s3.amazonaws.com/irs-form-990/201413219349308226_public.xml\n",
      "UNIVERSALGIVING 9\n",
      "https://s3.amazonaws.com/irs-form-990/201343199349309289_public.xml\n",
      "UNIVERSALGIVING 8\n",
      "https://s3.amazonaws.com/irs-form-990/201421819349300017_public.xml\n",
      "WISHBONEORG 6\n",
      "https://s3.amazonaws.com/irs-form-990/201402279349301040_public.xml\n",
      "WATSI INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201411139349300116_public.xml\n",
      "Wikimedia Foundation Inc 8\n",
      "https://s3.amazonaws.com/irs-form-990/201402309349300725_public.xml\n",
      "WORLDREADERORG 10\n",
      "https://s3.amazonaws.com/irs-form-990/201421709349300012_public.xml\n",
      "ZIDISHA INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201413169349300111_public.xml\n",
      "iCivics Inc 16\n",
      "https://s3.amazonaws.com/irs-form-990/201413179349302161_public.xml\n",
      "myAgro Farms 4\n",
      "https://s3.amazonaws.com/irs-form-990/201532099349200018_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE 1947 PARTITION ARCHIVE 2\n",
      "https://s3.amazonaws.com/irs-form-990/201502249349302085_public.xml\n",
      "BENEFICENT TECHNOLOGY INC DBA Benetech 16\n",
      "https://s3.amazonaws.com/irs-form-990/201501749349300500_public.xml\n",
      "BRACKETS FOR GOOD INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201511279349302026_public.xml\n",
      "THE CENTER TO PROMOTE HEALTHCARE ACCESS INC 18\n",
      "https://s3.amazonaws.com/irs-form-990/201512469349300951_public.xml\n",
      "CODEORG 13\n",
      "https://s3.amazonaws.com/irs-form-990/201502739349300730_public.xml\n",
      "CONNECT TO COMPETE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201502539349200750_public.xml\n",
      "year formed\n",
      "n employees\n",
      "COWORKERORG 5\n",
      "https://s3.amazonaws.com/irs-form-990/201520479349302657_public.xml\n",
      "CANCER COMMONS 9\n",
      "https://s3.amazonaws.com/irs-form-990/201500799349200305_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CAREERVILLAGE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201540429349300024_public.xml\n",
      "CENTER FOR STUDENT OPPORTUNITY INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201542379349200034_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CRISIS TEXT LINE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201542219349200209_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CRISIS TEXT LINE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201542219349300409_public.xml\n",
      "CRISIS TEXT LINE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201541409349300039_public.xml\n",
      "DO SOMETHING INC 16\n",
      "https://s3.amazonaws.com/irs-form-990/201521049349300137_public.xml\n",
      "DESIGN THAT MATTERS INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201501319349302305_public.xml\n",
      "THE GET SCHOOLED FOUNDATION 10\n",
      "https://s3.amazonaws.com/irs-form-990/201530309349300318_public.xml\n",
      "HARMONY INSTITUTE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201531279349300963_public.xml\n",
      "HOLLABACK INC 20\n",
      "https://s3.amazonaws.com/irs-form-990/201521559349300607_public.xml\n",
      "Impact Network International Inc 10\n",
      "https://s3.amazonaws.com/irs-form-990/201531359349101253_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE KIVA FOUNDATION 3\n",
      "https://s3.amazonaws.com/irs-form-990/201532179349200013_public.xml\n",
      "year formed\n",
      "n employees\n",
      "KANGU INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201532179349300428_public.xml\n",
      "KANGU INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201532249349302008_public.xml\n",
      "KHAN ACADEMY INC 18\n",
      "https://s3.amazonaws.com/irs-form-990/201522249349300707_public.xml\n",
      "LEARN FRESH EDUCATION CO 4\n",
      "https://s3.amazonaws.com/irs-form-990/201512529349300111_public.xml\n",
      "Lifebox Foundation Inc 6\n",
      "https://s3.amazonaws.com/irs-form-990/201531759349300018_public.xml\n",
      "THE LITERACY LAB 8\n",
      "https://s3.amazonaws.com/irs-form-990/201532649349300053_public.xml\n",
      "MEN OF COURAGE FOUNDATION DBA UNCOMMEN 4\n",
      "https://s3.amazonaws.com/irs-form-990/201511469349200336_public.xml\n",
      "year formed\n",
      "n employees\n",
      "The MakeSense Foundation 6\n",
      "https://s3.amazonaws.com/irs-form-990/201510689349300626_public.xml\n",
      "Moneythink 8\n",
      "https://s3.amazonaws.com/irs-form-990/201540369349300029_public.xml\n",
      "NEW CLASSROOMS INNOVATION PARTNERSINC 17\n",
      "https://s3.amazonaws.com/irs-form-990/201520449349302512_public.xml\n",
      "NATIONAL INSTITUTE ON MONEY IN STATE POLITICS 10\n",
      "https://s3.amazonaws.com/irs-form-990/201522619349300817_public.xml\n",
      "OCEARCH 11\n",
      "https://s3.amazonaws.com/irs-form-990/201501359349303045_public.xml\n",
      "THE OPENGOV FOUNDATION 8\n",
      "https://s3.amazonaws.com/irs-form-990/201531349349307138_public.xml\n",
      "PARTICIPATORY POLITICS FOUNDATION 4\n",
      "https://s3.amazonaws.com/irs-form-990/201500759349201000_public.xml\n",
      "year formed\n",
      "n employees\n",
      "REALLOCATE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201531629349300818_public.xml\n",
      "ReadworksInc 23\n",
      "https://s3.amazonaws.com/irs-form-990/201511039349300831_public.xml\n",
      "REASONING MIND INC 26\n",
      "https://s3.amazonaws.com/irs-form-990/201512119349100301_public.xml\n",
      "year formed\n",
      "n employees\n",
      "THE STELLAR FOUNDATION 5\n",
      "https://s3.amazonaws.com/irs-form-990/201512299349302171_public.xml\n",
      "SCIENCE BUDDIES 7\n",
      "https://s3.amazonaws.com/irs-form-990/201531359349306948_public.xml\n",
      "TECHSOUP GLOBAL 22\n",
      "https://s3.amazonaws.com/irs-form-990/201530239349300148_public.xml\n",
      "TIDEPOOL PROJECT 7\n",
      "https://s3.amazonaws.com/irs-form-990/201532239349201173_public.xml\n",
      "year formed\n",
      "n employees\n",
      "TRANSPARENCY TOOLKIT INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201510359349300431_public.xml\n",
      "Video Volunteers 10\n",
      "https://s3.amazonaws.com/irs-form-990/201532299349302913_public.xml\n",
      "WISHBONEORG 5\n",
      "https://s3.amazonaws.com/irs-form-990/201522269349303777_public.xml\n",
      "WATSI INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201530429349200523_public.xml\n",
      "year formed\n",
      "n employees\n",
      "weTHRIVE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201511199349300201_public.xml\n",
      "Wikimedia Foundation Inc 18\n",
      "https://s3.amazonaws.com/irs-form-990/201511419349300016_public.xml\n",
      "WORLDREADERORG 9\n",
      "https://s3.amazonaws.com/irs-form-990/201522179349301142_public.xml\n",
      "ZIDISHA INC 3\n",
      "https://s3.amazonaws.com/irs-form-990/201541339349303074_public.xml\n",
      "EDX INC 19\n",
      "https://s3.amazonaws.com/irs-form-990/201640419349301824_public.xml\n",
      "ADOPT-A-CLASSROOM INC 15\n",
      "https://s3.amazonaws.com/irs-form-990/201600929349300610_public.xml\n",
      "THE CENTER TO PROMOTE HEALTHCARE ACCESS INC 20\n",
      "https://s3.amazonaws.com/irs-form-990/201503179349300200_public.xml\n",
      "CODE-TO-LEARN FOUNDATION 6\n",
      "https://s3.amazonaws.com/irs-form-990/201621069349301122_public.xml\n",
      "CANCER COMMONS 8\n",
      "https://s3.amazonaws.com/irs-form-990/201630159349200408_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CAREERVILLAGE INC 6\n",
      "https://s3.amazonaws.com/irs-form-990/201513209349101051_public.xml\n",
      "year formed\n",
      "n employees\n",
      "CASE COMMONS INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201503489349301155_public.xml\n",
      "CENTER FOR STUDENT OPPORTUNITY INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201513149349303286_public.xml\n",
      "CLASSROOM INC 17\n",
      "https://s3.amazonaws.com/irs-form-990/201513179349305516_public.xml\n",
      "CODE FOR AMERICA LABS INC 17\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349314685_public.xml\n",
      "COMMON SENSE MEDIA 35\n",
      "https://s3.amazonaws.com/irs-form-990/201533209349311808_public.xml\n",
      "D-REV DESIGN FOR THE OTHER 90 12\n",
      "https://s3.amazonaws.com/irs-form-990/201523089349301407_public.xml\n",
      "DONORSCHOOSEORG 25\n",
      "https://s3.amazonaws.com/irs-form-990/201543179349202514_public.xml\n",
      "year formed\n",
      "n employees\n",
      "DTWO LTD 5\n",
      "https://s3.amazonaws.com/irs-form-990/201513209349303006_public.xml\n",
      "DEMOCRACY WORKS INC 12\n",
      "https://s3.amazonaws.com/irs-form-990/201630479349301353_public.xml\n",
      "DIGITAL GREEN FOUNDATION 7\n",
      "https://s3.amazonaws.com/irs-form-990/201543209349306209_public.xml\n",
      "E-THE PEOPLE 4\n",
      "https://s3.amazonaws.com/irs-form-990/201513249349200521_public.xml\n",
      "year formed\n",
      "n employees\n",
      "Elephant Action League 3\n",
      "https://s3.amazonaws.com/irs-form-990/201513249349200521_public.xml\n",
      "year formed\n",
      "n employees\n",
      "Elephant Action League 3\n",
      "https://s3.amazonaws.com/irs-form-990/201542809349300109_public.xml\n",
      "FAMILIES EMPOWERED 2\n",
      "https://s3.amazonaws.com/irs-form-990/201503149349301110_public.xml\n",
      "THE FREECYCLE NETWORK 3\n",
      "https://s3.amazonaws.com/irs-form-990/201502859349301150_public.xml\n",
      "GIVEWELL COMMUNITY FOUNDATION INC 22\n",
      "https://s3.amazonaws.com/irs-form-990/201523209349206167_public.xml\n",
      "year formed\n",
      "n employees\n",
      "GLOBAL LIVES PROJECT INC 11\n",
      "https://s3.amazonaws.com/irs-form-990/201513169349306556_public.xml\n",
      "GOOD WORLD SOLUTIONS INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201523229349300127_public.xml\n",
      "Great NonProfits 12\n",
      "https://s3.amazonaws.com/irs-form-990/201513209349306931_public.xml\n",
      "HOPING HEARTS FOUNDATION INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201543159349303854_public.xml\n",
      "HARMONY INSTITUTE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201611169349301116_public.xml\n",
      "HOLLABACK INC 10\n",
      "https://s3.amazonaws.com/irs-form-990/201543169349100614_public.xml\n",
      "year formed\n",
      "n employees\n",
      "HopeLab Foundation Inc 6\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349311725_public.xml\n",
      "IN OUR BACKYARDS INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201543159349303544_public.xml\n",
      "year formed\n",
      "INTERNET SEXUALITY INFORMATION SERVICES 9\n",
      "https://s3.amazonaws.com/irs-form-990/201513179349201336_public.xml\n",
      "year formed\n",
      "n employees\n",
      "IMAGING THE WORLD CORP 10\n",
      "https://s3.amazonaws.com/irs-form-990/201523209349311202_public.xml\n",
      "InSTEDD 6\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349316710_public.xml\n",
      "International Bridges to Justice Inc 13\n",
      "https://s3.amazonaws.com/irs-form-990/201503009349301060_public.xml\n",
      "INTERNEWS NETWORK INC 22\n",
      "https://s3.amazonaws.com/irs-form-990/201513209349312361_public.xml\n",
      "Literacy Bridge 7\n",
      "https://s3.amazonaws.com/irs-form-990/201543149349302334_public.xml\n",
      "year formed\n",
      "The Lunchbox Fund 4\n",
      "https://s3.amazonaws.com/irs-form-990/201620259349301152_public.xml\n",
      "MIND Research Institute 33\n",
      "https://s3.amazonaws.com/irs-form-990/201513209349310671_public.xml\n",
      "MEDIC MOBILE INC 8\n",
      "https://s3.amazonaws.com/irs-form-990/201533209349311223_public.xml\n",
      "MEDIC MOBILE INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201513079349301056_public.xml\n",
      "Moneythink 7\n",
      "https://s3.amazonaws.com/irs-form-990/201523209349312592_public.xml\n",
      "MOZILLA FOUNDATION 18\n",
      "https://s3.amazonaws.com/irs-form-990/201630749349300613_public.xml\n",
      "NEW CLASSROOMS INNOVATION PARTNERSINC 17\n",
      "https://s3.amazonaws.com/irs-form-990/201513559349300081_public.xml\n",
      "NATIONAL INSTITUTE ON MONEY IN STATE POLITICS 10\n",
      "https://s3.amazonaws.com/irs-form-990/201523209349312412_public.xml\n",
      "year formed\n",
      "NEXLEAF ANALYTICS 9\n",
      "https://s3.amazonaws.com/irs-form-990/201523219349300537_public.xml\n",
      "NPOWER INC 27\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349310235_public.xml\n",
      "OCEARCH 9\n",
      "https://s3.amazonaws.com/irs-form-990/201543209349314394_public.xml\n",
      "OPEN MEDIA FOUNDATION 11\n",
      "https://s3.amazonaws.com/irs-form-990/201503139349302785_public.xml\n",
      "OPERATION ASHA NFP 12\n",
      "https://s3.amazonaws.com/irs-form-990/201513159349201371_public.xml\n",
      "year formed\n",
      "n employees\n",
      "Planetwork NGO Inc 4\n",
      "https://s3.amazonaws.com/irs-form-990/201640509349200504_public.xml\n",
      "year formed\n",
      "n employees\n",
      "PLANETWORK NGO INC 4\n",
      "https://s3.amazonaws.com/irs-form-990/201512589349300801_public.xml\n",
      "PARTICIPATORY CULTURE FOUNDATION INC 7\n",
      "https://s3.amazonaws.com/irs-form-990/201543169349302424_public.xml\n",
      "PULSEPOINT FOUNDATION 6\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349206655_public.xml\n",
      "year formed\n",
      "n employees\n",
      "REALLOCATE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201610559349300016_public.xml\n",
      "REASONING MIND INC 25\n",
      "https://s3.amazonaws.com/irs-form-990/201512949349300511_public.xml\n",
      "Single Stop USA Inc 20\n",
      "https://s3.amazonaws.com/irs-form-990/201502799349300705_public.xml\n",
      "SkyTruth 8\n",
      "https://s3.amazonaws.com/irs-form-990/201503209349304545_public.xml\n",
      "TEACHAIDS 5\n",
      "https://s3.amazonaws.com/irs-form-990/201601379349310585_public.xml\n",
      "TECHSOUP GLOBAL 23\n",
      "https://s3.amazonaws.com/irs-form-990/201630129349300623_public.xml\n",
      "TIDEPOOL PROJECT 6\n",
      "https://s3.amazonaws.com/irs-form-990/201533219349300518_public.xml\n",
      "UNIVERSALGIVING 10\n",
      "https://s3.amazonaws.com/irs-form-990/201610149349300801_public.xml\n",
      "Video Volunteers 13\n",
      "https://s3.amazonaws.com/irs-form-990/201631249349300518_public.xml\n",
      "WITNESS INC 19\n",
      "https://s3.amazonaws.com/irs-form-990/201620419349200622_public.xml\n",
      "year formed\n",
      "n employees\n",
      "weTHRIVE INC 5\n",
      "https://s3.amazonaws.com/irs-form-990/201621269349301537_public.xml\n",
      "Wikimedia Foundation Inc 19\n",
      "https://s3.amazonaws.com/irs-form-990/201610929349300631_public.xml\n",
      "ZEARN INC 14\n",
      "https://s3.amazonaws.com/irs-form-990/201611279349300401_public.xml\n",
      "edX Inc 21\n",
      "https://s3.amazonaws.com/irs-form-990/201513219349301356_public.xml\n",
      "iCivics Inc 14\n",
      "https://s3.amazonaws.com/irs-form-990/201620539349300532_public.xml\n",
      "myAgro Farms 9\n"
     ]
    }
   ],
   "source": [
    "# pull 990 xmls\n",
    "metadata_df = pd.read_csv('output/metadata.csv', header=0)\n",
    "rows = []\n",
    "\n",
    "for xmlid in metadata_df['OBJECT_ID']:\n",
    "    # pull from S3\n",
    "    url = 'https://s3.amazonaws.com/irs-form-990/%d_public.xml' % xmlid\n",
    "    print url\n",
    "    response = requests.get(url)\n",
    "    \n",
    "    # parse to soup\n",
    "    soup = BeautifulSoup(response.text[3:], 'xml')\n",
    "    \n",
    "    # get org data\n",
    "    ein = soup.find('EIN').contents[0]\n",
    "    try:\n",
    "        org_name = soup.find('Filer').Name.BusinessNameLine1.contents[0]\n",
    "    except AttributeError:\n",
    "        try:\n",
    "            org_name = soup.find('Filer').BusinessName.BusinessNameLine1.contents[0]\n",
    "        except AttributeError:\n",
    "            org_name = soup.find('Filer').BusinessName.BusinessNameLine1Txt.contents[0]\n",
    "    try:\n",
    "        city = soup.find('Filer').USAddress.City.contents[0]\n",
    "    except AttributeError:\n",
    "        city = soup.find('Filer').USAddress.CityNm.contents[0]\n",
    "    try:\n",
    "        state = soup.find('Filer').USAddress.State.contents[0]\n",
    "    except AttributeError:\n",
    "        state = soup.find('Filer').USAddress.StateAbbreviationCd.contents[0]\n",
    "    try:\n",
    "        tax_year = soup.find('TaxYear').contents[0]\n",
    "    except AttributeError:\n",
    "        tax_year = soup.find('TaxYr').contents[0]\n",
    "    try:\n",
    "        submission_time = soup.find('Timestamp').contents[0]\n",
    "    except AttributeError:\n",
    "        submission_time = soup.find('ReturnTs').contents[0]        \n",
    "    try:\n",
    "        year_formed = soup.find('YearFormation').contents[0]\n",
    "    except AttributeError:\n",
    "        try:\n",
    "            year_formed = soup.find('FormationYr').contents[0]\n",
    "        except:\n",
    "            print 'year formed'\n",
    "            year_formed = None\n",
    "    try:\n",
    "        n_employees = soup.find('TotalNbrEmployees').contents[0]\n",
    "    except AttributeError:\n",
    "        try:\n",
    "            n_employees = soup.find('TotalEmployeeCnt').contents[0]\n",
    "        except:\n",
    "            print 'n employees'\n",
    "            n_employees = None\n",
    "\n",
    "    # get revenue\n",
    "    total_revenue = None\n",
    "    revenue_fields = ['TotalRevenueCurrentYear', 'TotalRevenue', 'TotalRevenueAmt',\n",
    "                      'TotalRevenueAndExpenses', 'CYTotalRevenueAmt', 'TotalRevAndExpnssAmt']\n",
    "    for revenue_field in revenue_fields:\n",
    "        try:\n",
    "            total_revenue = soup.find(revenue_field).contents[0]\n",
    "            break\n",
    "        except AttributeError:\n",
    "            pass\n",
    "    if total_revenue in [None, '\\n']:\n",
    "        break\n",
    "        \n",
    "    base_row = [\n",
    "        org_name, city, state, ein, url,\n",
    "        tax_year, year_formed, submission_time,\n",
    "        n_employees, total_revenue,\n",
    "    ]\n",
    "\n",
    "    # get compensation data\n",
    "    n_people_found = 0\n",
    "    for employee in soup.find_all('Form990PartVIISectionA'):\n",
    "        try:\n",
    "            title = employee.Title.contents[0]\n",
    "            comp = employee.ReportableCompFromOrganization.contents[0]\n",
    "        except AttributeError:\n",
    "            continue\n",
    "        rows.append(base_row + [title, comp])\n",
    "        n_people_found += 1\n",
    "    for employee in soup.find_all('Form990PartVIISectionAGrp'):\n",
    "        try:\n",
    "            title = employee.TitleTxt.contents[0]\n",
    "            comp = employee.ReportableCompFromOrgAmt.contents[0]\n",
    "        except AttributeError:\n",
    "            continue\n",
    "        rows.append(base_row + [title, comp])\n",
    "        n_people_found += 1\n",
    "    for employee in soup.find_all('OfficerDirTrstKeyEmplGrp') + soup.find_all('OfficerDirectorTrusteeEmplGrp'):\n",
    "        try:\n",
    "            title = employee.TitleTxt.contents[0]\n",
    "            comp = employee.CompensationAmt.contents[0]\n",
    "        except AttributeError:\n",
    "            continue\n",
    "        rows.append(base_row + [title, comp])\n",
    "        n_people_found += 1\n",
    "    for employee in soup.find_all('OfficerDirectorTrusteeKeyEmpl') + soup.find_all('OfcrDirTrusteesOrKeyEmployee') + soup.find_all('CompensationOfHighestPaidEmpl'):\n",
    "        try:\n",
    "            title = employee.Title.contents[0]\n",
    "            comp = employee.Compensation.contents[0]\n",
    "        except AttributeError:\n",
    "            continue\n",
    "        rows.append(base_row + [title, comp])\n",
    "        n_people_found += 1\n",
    "\n",
    "    print org_name, n_people_found\n",
    "\n",
    "df = pd.DataFrame(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# label and output\n",
    "df.columns = [\n",
    "    'org_name', 'city', 'state', 'ein', 'url',\n",
    "    'tax_year', 'year_formed', 'submission_time',\n",
    "    'n_employees', 'total_revenue',\n",
    "    'title', 'comp',\n",
    "]\n",
    "df.to_csv('output/comp_data_raw.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'Executive Director' u'Dir of App Solutions' u'director'\n",
      " u'Product Manager' u'asst dir of app solution' u'Dir Fin & Admin Thru 4/6'\n",
      " u'Exec Director/CEO' u'Board Member' u'CHAIRMAN' u'TREASURER' u'SECRETARY'\n",
      " u'DIRECTOR' u'PRESIDENT' u'VP AND COO' u'VP OF CURRICULUM & ASSESSMENT'\n",
      " u'DIRECTOR OF FINANCE' u'SENIOR DIRECTOR OF BUSINESS DEVELOPMENT'\n",
      " u'Director' u'Executive Direc' u'CHAIRMAN, TREASURER'\n",
      " u'CHIEF EXECUTIVE OFFICER' u'Founder' u'Board Chair' u'VICE CHAIR'\n",
      " u'CHAIRPERSON' u'VICE PRESIDENT' u'President & CEO' u'Dir of Prod Dev'\n",
      " u'VP Impact & Adm' u'VP Staff Dev' u'VP Strat. P/S' u'VP Research'\n",
      " u'COFOUNDER & EXECUTIVE DIRECTOR' u'COFOUNDER & CHIEF OPERATING OFFICER'\n",
      " u'COFOUNDER, DIRECTOR' u'CHAIR, BOARD OF DIRECTOR'\n",
      " u'SECRETARY, BOARD OF DIRECTORS' u'TRESURER, BOARD OF DIRECTORS'\n",
      " u'BOARD OF DIRECTORS' u'Trustee' u'President' u'Secretary' u'Treasurer'\n",
      " u'TRUSTEE' u'PRESIDENT &' u'VICE PRES' u'CHAIRMAN/CEO' u'CFO'\n",
      " u'VP AFRICA MKTG & OPERATIONS' u'VP OF LEARNING' u'VICE-PRESIDENT'\n",
      " u'TREASURER & EXEC. DIR.' u'EXECUTIVE DIRECTOR' u'BOARD CHAIR'\n",
      " u'BOARD MEMBER' u'CHIEF OPERATING OFFICER' u'CHIEF DELIVERY OFFICER'\n",
      " u'Director, President' u'Director, Chairman'\n",
      " u'Director, Chief Technology Officer' u'Secretary, Treasurer'\n",
      " u'PRESIDENT & CO-CEO' u'CO-CEO' u'CONTROLLER'\n",
      " u'SENIOR DIR. BUS. DEVELOPMENT' u'COO - IT AND OPERATIONS'\n",
      " u'VP, DEVELOPMENT' u'VP, TECHSOUP PRODUCTS' u'CEO, Treasurer, Director'\n",
      " u'Director/Board chair' u'Chairman' u'Treasurer/Director' u'Chair'\n",
      " u'Vice-Chair' u'Executive Secretary' u'Trustee (Founder)' u'CFOO'\n",
      " u'General Counsel' u'President; Former Executive Director'\n",
      " u'Treasurer/Secretary' u'CHAIR' u'CEO' u'Vice Chairman'\n",
      " u'SENIOR DIRECTOR OF BUSINESS' u'PRESIDENT/COO' u'INTERIM PRESIDENT/COO'\n",
      " u'EDITOR-IN-CHIEF' u'CHIEF EDUCATION & STRATEGY OFFICER'\n",
      " u'VP OF MARKETING' u'VP OF EDUCATION CONTENT' u'CHIEF DEVELOPMENT OFFICER'\n",
      " u'MANAGING DIR, EDUCATION & RESEARCH' u'DIRECTOR OF PRODUCT DEVELOPMENT'\n",
      " u'VICE PRESIDE' u'FORMER PRESIDENT' u'PRESIDENT/FORMER VICE PRESIDENT'\n",
      " u'SECRETARY/TREASURER' u'Vice Chair' u'Program Manager'\n",
      " u'Board Chair and Director' u'Secretary and Director'\n",
      " u'VP of Communications, Treasurer, and Director' u'Vice President'\n",
      " u'VP Comm & Mktg' u'PRESIDENT/CPO' u'CTO' u'COO' u'SENIOR PROGRAM MGR'\n",
      " u'Executive Director and Officer' u'Deputy Director' u'CHIEF TECH. OFF'\n",
      " u'Pres-Education' u'Pres-Research' u'VP-ED SVCS' u'VP-MARKETING'\n",
      " u'VP-SALES' u'VP-DEVELOPMENT' u'CIO' u\"Sec'y/Treasurer\" u'SALES ASSOCIATE'\n",
      " u'SALES DIRECTOR' u'PRESIDENT/DIRECTOR' u'TREASURER/SECRETARY/DIRECTOR'\n",
      " u'TRUSTEE/DIRECTOR' u'PRESIDENT, DIRECTOR' u'SECRETARY, TREAS, DIRECTOR'\n",
      " u'VP AFRICA MKTG & OPERATION' u'SERVER ARCHITECT'\n",
      " u'DIR LEARNING SOUTH AFRICA' u'CHAIRMAN OF THE BOARD' u'VP Fin & Admin'\n",
      " u'Chief Tech Officer' u'Senior VP' u'VP Strategy' u'Dir. Software Dev.'\n",
      " u'PRESIDENT AND CO-CEO' u'CHIEF FINANCIAL OFFICER' u'VICE PRESIDENT-TSP'\n",
      " u'COO-PLANNING AND DEPLOYMENT' u'SR. DIRECTOR, BIZ DEV CBDO'\n",
      " u'CEO, Director' u'PRESIDENT AND BOARD MEMBER'\n",
      " u'Vice-Chair (Thru July 2010)' u'Treasurer & Vice Chair'\n",
      " u'Chief Technology Officer' u'Chief Community Officer' u'Project Director'\n",
      " u'CEO AND CO-FOUNDER' u'MANAGING DIRECTOR AND CO-FOUNDER'\n",
      " u'BOARD DIRECTOR' u'Director; Former Executiv' u'Executive Dir.'\n",
      " u'DIRECTOR (THRU 6/30/12)' u'CO-EXECUTIVE DIRECTOR'\n",
      " u'DIR OF APP SOLUTIONS' u'ASST DIR OF APPLICATION' u'CHAIR OF THE BOARD'\n",
      " u'CEO/BOARD MEMBER' u'SECRETARY/BOARD MEMBER' u'TREASURER/BOARD MEMBER'\n",
      " u\"GEN'L COUNSEL & DIR OF POL\" u'DIR/PRODUCT DEVELOPM'\n",
      " u'DIRECTOR OF ENGINEER' u'PROJECT DESIGN LEAD' u'SR. SOFTWARE ENGINEE'\n",
      " u'PROJECT EXECUTIVE' u'PROGRAM DIR.' u'PRGM DIR/DIR.'\n",
      " u'VP OF FINANCE AND ADMINISTRATION'\n",
      " u'CHIEF EDUCATION AND STRATEGY OFFICER' u'GENERAL MANAGER OF EDUCATION'\n",
      " u'CHIEF TECHNOLOGY OFFICER' u'VP OF PRODUCT DEVELOPMENT' u'VP OF POLICY'\n",
      " u'MANAGING DIR., EDUCATION AND RESEARCH' u'SENIOR ENGINEER'\n",
      " u'VP OF COMMUNICATION' u'FOUNDER' u'DIRECTOR OF OPERATIONS'\n",
      " u'DIRECTOR OF ANALYTICS' u'SECRETARY OF CORPORATION' u'CEO and Member'\n",
      " u'Member' u'DEPUTY DIRECTOR' u'MEMBER' u'Dir Prod Dev' u'VP Strat Ptrshps'\n",
      " u'COFOUNDER & BOARD MEMBER' u'COFOUNDER & EXECUTIVE DIRE'\n",
      " u'COFOUNDER & CHIEF OPERATIN' u'COFOUNDER & CHIEF PARTNERSHIPS OFFICER'\n",
      " u'Executive Vice President' u'VP of Bus. Strategy' u'Co-chair'\n",
      " u'Sr. Vice President' u'Country Director' u'FOUNDER & CEO'\n",
      " u'PRESIDENT & COO' u'HEAD OF FINANCE' u'LEAD DEVELOPER' u'LEAD DESIGNER'\n",
      " u'HEAD OF STRATEGY & PROGRAMS' u'LEAD EXERCISES DEVELOPER'\n",
      " u'SOFTWARE ENGINEER' u'DEAN OF COMPUTER SCIENCE' u'VP SALES'\n",
      " u'CHEIF PROGRAM OFFICER' u'PEXECUTIVE DIRECTOR' u'VICE CHAIRMAN'\n",
      " u'DIRECTOR OF FINANCE & ADMINISTRATION' u'SR. MANAGER'\n",
      " u'FOUNDER/BOARD CHAIR' u'CFO/BOARD MEMBER' u'CFAO' u'VP Programs'\n",
      " u'SR. DIR & CHIEF BIZ DVL' u'VICE PRESIDENT DEVELOPMENT'\n",
      " u'SR DIR. INFRASTRUCTURE' u'CFA' u'Chief Revenue Officer'\n",
      " u'Chief Global Development OFC' u'Chief Talent and Culture OFC'\n",
      " u'PRESIDENT AND FOUNDER' u'CEO/Co-Founder'\n",
      " u'BOARD MEMBER/SEE SCH O FOR COMP INFOP' u'BOARD MEMBER/INTERIM CEO'\n",
      " u'DIRECTOR/DIR APPLICATION SOLUTIONS' u'ASST DIR APPLICATION SOLUTIONS'\n",
      " u'SR SOLUTIONS LEAD' u'PRESIDENT/CHAIR' u'VICE-CHAIR'\n",
      " u'ASSISTANT SECRETARY' u'PRESIDENT/CEO' u'BOARD SECRETARY/COO'\n",
      " u'TREASURER/DIRECTOR OF FINANCE & ADMINISTRATION' u'DIRECTOR OF EDUCATION'\n",
      " u'CHIEF EVANGELIST' u'DIRECTOR, PAST PRESIDENT' u'DIRECTOR SECRETARY'\n",
      " u'DIRECTOR, TREASURER' u'PROGRAM DIRECTOR' u'PRESIDENT AND TREASURER'\n",
      " u'EXECUTIVE DIRECTOR AND CLERK' u'TERM ENDED OCT. 2013'\n",
      " u'VP OF FINANCE & ADMINISTRATION' u'SENIOR EDUCATION ADVISOR'\n",
      " u'VP OF DEVELOPMENT' u'VP OF BUSINESS DEVELOPMENT' u'CO EXEC DIR'\n",
      " u'BOARD MEMBER (THRU 12/31/2013)' u'BOARD MEMBER (THRU 5/31/2013)'\n",
      " u'EDUCATION VP AND GENERAL MANAGER' u'CO DIRECTOR OF ENGINEERING'\n",
      " u'BOARD CHAIRMAN' u'DIRECTOR OF TECHNICAL OPERATIONS'\n",
      " u'CHIEF PARTNERSHIPS OFFICER' u'CHIEF MARKETING OFFICER'\n",
      " u'SENIOR VICE PRESIDENT' u'DIRECTOR OF ENGINEERING'\n",
      " u'PRINCIPAL SOFTWARE ENGINEER' u'EXEC DIRECTO' u'SECRETARY, T'\n",
      " u'chairperson' u'FORMER EXECUTIVE DIRECTOR' u'BD CHAIR / ED'\n",
      " u'President and CEO' u'PRESIDENT/EXECUTIVE DIRECTOR'\n",
      " u'PRESIDENT/EXECUTIVE DIRECT' u'COFOUNDER & CHIEF PARTNERS'\n",
      " u'BOARD MEMBER - THRU 3/2013' u'TREASURER/FINANCE CHAIR'\n",
      " u'BOARD MEMBER/FUNDRAISING CHAIR' u'SECRETARY - THRU 3/2013'\n",
      " u'EXECUTIVE DI' u'CHAIR AND V.' u'President, CEO' u'Board Member, COO'\n",
      " u'Board Member. Chief Tech Officer' u'Dir. of Engineering'\n",
      " u'Project Manager' u'President Emeritus/Founder' u'President/CEO'\n",
      " u'Senior Vice President' u'ED - ICCIL' u'VP - Asia and Environment'\n",
      " u'VP - ICT Policy and Programs' u'COP - South Sudan'\n",
      " u'Vice President - Finance' u'FOUNDER & EXECUTIVE DIRECTOR'\n",
      " u'HEAD OF STRATEGY AND PROGRAMS' u'PROGRAM MANAGER'\n",
      " u'PRESIDENT / DIRECTOR' u'TRESURER / DIRECTOR' u'CLERK / DIRECTOR'\n",
      " u'CO DIRECTOR' u'VP Product Dvlp' u'EMPLOYEE' u'BOARD OF DIRECTOR'\n",
      " u'CHIEF STRATEGIST' u'CHIEF PROGRAM OFFICER' u'THROUGH 2/1/2013'\n",
      " u'CHIEF ACADEMIC OFFICER' u'DIRECTOR OF ARCHITECTURE'\n",
      " u'DIR. OF CENTRAL PROG. OPS.' u'VICE CHAIRMAN OF THE BOARD'\n",
      " u'VP BUSINESS DEVELOPMENT' u'VP HARDWARE' u'CHAIRMAN ON THE BOARD'\n",
      " u'DIRECTOR FORMER EXECUTIVE DIRECTOR & TREASURER'\n",
      " u'CURRENT EXECUTIVE DIRECTOR & TREASURER' u'CEO & PRESIDENT'\n",
      " u'SENIOR VP OF NATIONAL EXPANSION' u'VP OF SOFTWARE DEVELOPMENT' u'VP-CFO'\n",
      " u'VP OF PROGRAM' u'HEAD OF MOSCOW BRANCH' u'VP OF CURRICULUM DEVELOPMENT'\n",
      " u'VP OF SALES' u'SR. SOFTWARE ENGINEER' u'CHIEF SOFTWARE ARC'\n",
      " u'QC/QA MANAGER' u'Chief Prog Officer' u'VP FINANCE'\n",
      " u'VP, ALLIANCES & GLOBAL MEDIA' u'SR. CUSTOM PRO SPECIALIST'\n",
      " u'VP COMMUNITY & PLATFORMS' u'Treasurer and CFA'\n",
      " u'Secretary and General Counsel' u'Chief Talent & Culture Officer'\n",
      " u'dir of engineering operations' u'Deputy General Counsel'\n",
      " u'PRESIDENT & CEO' u'VICE PRESIDENT & SECRETARY'\n",
      " u'BOARD DIRECTOR (FROM 10/13)' u'BOARD DIRECTOR (UNTIL 10/13)'\n",
      " u'DIRECTOR, FINANCE & CONTROL' u'E.D. through July 2013'\n",
      " u'President and E.D.' u'Treasurer & Secretary' u'Pres & Dir'\n",
      " u'Treas & Dir' u'Sec & Dir' u'Intl Fin Dir' u'VP & GM, LITERACY'\n",
      " u'VP HR & ADMIN' u'DIRECTOR OF OPS' u'DIR PRODUCT STGY'\n",
      " u'DIR OF MARKETING' u'DIR CONTENT ACQUIS' u'DIRECTOR OF PROGRAMMING'\n",
      " u'CORPORATE SECRETARY' u'DIRECTOR OF INFRASTRUCTURE'\n",
      " u'DIR APPLICATION SOLUTIONS' u'ASST DIR APPLICATION SOLUT'\n",
      " u'CHIEF TECHNICAL ARCHITECT' u'PROJECT MANAGER'\n",
      " u'OPERATIONS SUPPORT MANAGER' u'SPECIAL PROJECTS MANAGER'\n",
      " u'CHIEF DATA ARCHITECT' u'TREASURER/VP OF FINANCE & ADMIN'\n",
      " u'VP OF EDUCATION' u'VP OF PRODUCT' u'ENGINEER' u'PRESIDENT AND CEO'\n",
      " u'CHAIR, DIRECTOR' u'DIRECTOR, SECRETARY' u'LEAD DIRECTOR'\n",
      " u'ENGINEERING DIRECTOR' u'CHIEF SCIENTIST'\n",
      " u'Board Member and Executive Director' u'DIRECTOR - 2014'\n",
      " u'EXEC DIRECTOR TO NOV 2014' u'DIRECTOR OF TECHNICAL PRODUCT' u'CMO'\n",
      " u'TREASURER - FORMER' u'TREASURER - CURRENT'\n",
      " u'BOARD CHAIR - FROM APRIL 2014' u'BOARD VICE-CHAIR - FROM APRIL 2014'\n",
      " u'TREASURER - FROM APRIL 2014' u'SECRETARY - FROM APRIL 2014'\n",
      " u'BOARD MEMBER - FROM APRIL 2014' u'BOARD MEMBER - THROUGH APRIL 2014'\n",
      " u'Director (from 07/15)' u'DIRECTOR (THRU 4/7/2014)'\n",
      " u'LEAD PRODUCT MANAGER' u'EXECUTIVE CH' u'VP-LIC & BUS' u'CO-DIRECTOR'\n",
      " u'Director thru Spring 2014' u'DIRECTOR/CHIEF EXECUTIVE OFFICER'\n",
      " u'DIRECTOR/CHIEF PROGRAM OFFICER' u'THROUGH APRIL 2013'\n",
      " u'DIRECTOR OF SITE OPERATIONS' u'DIRECTOR OF BASE OPERATIONS'\n",
      " u'DIRECTOR OF PROGRAM INITIATIVES' u'SOFTWARE DEVELOPER'\n",
      " u'EXECUTIVE VICE PRESIDENT' u'CHIEF SYSTEMS ARCHITECT'\n",
      " u'VP OF CURRICULUM DEVELOPME' u'FOUNDER & SECRETARY' u'VP - FINANCE'\n",
      " u'CEO - CARAVAN STUDIOS' u'VP - ALLIANCES & GLOBAL MED'\n",
      " u'SR. DIR. BIZ DEV. CBDO' u'VP - TECHSOUP GLOBAL NETWORK'\n",
      " u'VP - TECHNOLOGY SOLUTIONS' u'VP - STRATEGY & IMPACT' u'LEAD MEDICAL'\n",
      " u'VP PRODUCT &' u'VP ENGINEERI' u'SOFTWARE ENG' u'CLERK' u'founder'\n",
      " u'chair' u'trustee' u'Dir of Engineering Features'\n",
      " u'Dir of Platform Engineer' u'DIRECTOR, HUMAN & ORG.'\n",
      " u'Co-Chair & Director' u'Treasurer/CFO' u'VP and General Counsel'\n",
      " u'President, COO' u'VP of Strategic Partnerships' u'edX Architect'\n",
      " u'Software Architect' u'Director of PR and Communications'\n",
      " u'VP for Engineering' u'DIRECTOR OF HUMAN RESOURCES'\n",
      " u'DIRECTOR OF BUSINESS & PARTNERSHIP DEVELOPMENT' u'IT MANAGER'\n",
      " u'SR. NETWORK ADMINISTRATOR' u'TERM ENDED SEPT. 2014'\n",
      " u'CHIEF OF EDUCATION PARTNER' u'VP OF FINANCE & ADMINISTRA' u'CAO'\n",
      " u'CRO & CMO' u'PROGRAM DIRCTOR' u'PROGAM DIRECTOR' u'DESIGNER'\n",
      " u'BOARD MEMBER (THRU 11/2014)' u'VP OF MARKETING AND GM CONSUMER'\n",
      " u'VP EDITORIAL DIRECTOR' u'NEW YORK DIRECTOR'\n",
      " u'DIRECTOR OF TECHNICAL OPER' u'BOARD TREASU' u'SENIOR SOFTWARE ENGINEER'\n",
      " u'Director and President' u'Director and Treasurer'\n",
      " u'Director and Secretary' u'DIRECTOR OF FINANCE AND OPERATIONS'\n",
      " u'EXEC COMMITT' u'PRESIDENT/CE' u'IMMEDIATE PA' u'VICE CHAIRMA'\n",
      " u'Executive Director, Secretary' u'DIRECTOR/CHAIRMAN' u'PRES/TREAS'\n",
      " u'BOARD VICE-CHAIR' u'Treas,Board Mbr' u'Secretary, COO'\n",
      " u'COFOUNDER & CHIEF OPERATING' u'BOARD MEMBER-THRU 11/2014'\n",
      " u'BOARD MEMBER/SECRETARY' u'BOARD MEMBER/GOVERNANCE CHAIR'\n",
      " u'Interm Exec Dir' u'Sec./Treasurer, COO, CFO' u'Acting Chair'\n",
      " u'Board Member. CEO' u'BOARD MEMBER AND CLERK'\n",
      " u'Chief of Party - South Sudan' u'Sr. Vice President - IRL'\n",
      " u'Country Director - Afghanistan' u'VP Prog. - Europe, Eurasia & Asia'\n",
      " u'Director through May 2015' u'Director through June 2015'\n",
      " u'Director/CEO Emeritus' u'President, Research Division'\n",
      " u'Chief Strategist' u'Chief Executive Officer'\n",
      " u'CFO/Secretary & Treasurer' u'VP, Educ. Services' u'Regional VP'\n",
      " u'Chief Partnerships Officer' u'VP, Philanthropic Partnerships'\n",
      " u'VP, Emerging Markets' u'Chief of Staff' u'Dir., Strategic Partnerships'\n",
      " u'Strategic Partnerships Mgr.' u'Educ. Partnerships Mgr.'\n",
      " u'Executive Director/Secretary' u'EXECUTIVE DIRECTOR/PRESIDENT'\n",
      " u'SECRETARY/ VP OPERATIONS' u'VP LEARNING' u'VP PRODUCTS' u'VP ENGAGEMENT'\n",
      " u'DIRECTOR, PARTNERSHIPS' u'DIRECTOR, OPENNEWS'\n",
      " u'ONLINE ORGANIZING & FUNDRAISING LEAD' u'SR. DIRECTOR BADGES'\n",
      " u'DIRECTOR, HIVE CHICAGO' u'CHIEF GROWTH OFFICER'\n",
      " u'DATA SOLUTIONS ARCHITECT' u'VP OF PRODUCTION VISION'\n",
      " u'SENIOR DIRECTOR OF TALENT' u'SENIOR DIRECTOR OF PROGRAM INITIATIVES'\n",
      " u'Pres., Director' u'CFO/Sec/Direct' u'Chief Science Officer'\n",
      " u'VICE CHAIR & HEAD OF DEVELOPMENT COMMITTEE'\n",
      " u'TREASURER & HEAD OF AUDIT COMMITTEE'\n",
      " u'HEAD OF NOMINATING & GOVERNANCE COMMITTEE' u'DIRECTOR OF FINANCE & ADMI'\n",
      " u'DIRECTOR OF DEVELOPMENT' u'DIRECTOR-TECHNOLOGY SERVICE CORPS'\n",
      " u'DIRECTOR-THE COMMUNITY CORPS'\n",
      " u'REGIONAL DIRECTOR - TECHNOLOGY SERVICE CORPS'\n",
      " u'DIRECTOR FOUNDATION CONNECT' u'TREASURERCFO' u'CHAIRMAN-ELECT'\n",
      " u'COO AND VP OF FINANCE' u'COO/Gen Counsel' u'CDO' u'CSO'\n",
      " u'Managing Dir, NYC' u'VP - ALLIANCES & GLOBAL ME'\n",
      " u'VP - TECHSOUP GLOBAL NETWO' u'VP - TECHNOLOGY'\n",
      " u'VP - STRATEGIC ALLIANCES & GENERAL COUNSEL' u'vice chair'\n",
      " u'VP of Product' u'Special Advisor' u'Director of Engineering'\n",
      " u'Sr. dir. community engagement' u'CEO/PRESIDENT, DIRECTOR THRU 6/30/15'\n",
      " u'CPO/ED/SECRETARY' u'CHAIR AND DIRECTOR' u'TREASURER AND DIRECTOR'\n",
      " u'ACADEMIC DIRECTOR' u'INSTRUCTIONAL DESIGNER' u'PRODUCT DIRECTOR'\n",
      " u'President & COO' u'Clerk and General Counsel'\n",
      " u'Vice President of Strategic Partnerships'\n",
      " u'Vice President of Education Services' u'VP, Business Development'\n",
      " u'VP, Marketing' u'edX Senior Director, Engineering'\n",
      " u'President and E.D. to August 2014' u'Treasurer and Operations Manager'\n",
      " u'Secretary and Chief of Staff' u'Exec Director']\n"
     ]
    }
   ],
   "source": [
    "# full list of job titles\n",
    "print df['title'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# job title regex\n",
    "\n",
    "# CEO\n",
    "ceo_label = 'ceo'\n",
    "ceo_regex = ['exec', 'ceo', 'e\\.d\\.', 'ed$']\n",
    "ceo_antiregex = []\n",
    "\n",
    "# CTO\n",
    "cto_label = 'cto'\n",
    "cto_regex = ['tech[n. ]', '^cto', 'software', 'engin', 'product[s ]', 'product$',\n",
    "             'app', 'developer', 'archit', 'prod dev']\n",
    "cto_antiregex = ['regional', 'service']\n",
    "\n",
    "# COO\n",
    "coo_label = 'coo'\n",
    "coo_regex = ['operat', 'coo', 'ops']\n",
    "coo_antiregex = ['africa', 'engin']\n",
    "\n",
    "# CFO\n",
    "cfo_label = 'cfo'\n",
    "cfo_regex = ['fin', 'cfo', 'treas', 'reven', 'cfa', 'tres']  # sic\n",
    "cfo_antiregex = []\n",
    "\n",
    "# BD/partnerships\n",
    "bd_label = 'bd'\n",
    "bd_regex = ['partner', 'business', 'bus\\.', 'growth', 'cpo',\n",
    "            'biz', 'sales', 'ptr', 'p/s']\n",
    "bd_antiregex = []\n",
    "\n",
    "# president\n",
    "pres_label = 'pres'\n",
    "pres_regex = ['pres']\n",
    "pres_antiregex = ['vice']\n",
    "\n",
    "# marketing\n",
    "cmo_label = 'cmo'\n",
    "cmo_regex = ['marketing', 'cmo', 'pr ', 'communications']\n",
    "cmo_antiregex = []\n",
    "\n",
    "# development\n",
    "cdo_label = 'cdo'\n",
    "cdo_regex = ['development', 'cdo']\n",
    "cdo_antiregex = ['bus', 'software', 'product', 'curriculum', 'global']\n",
    "\n",
    "# board\n",
    "board_label = 'board'\n",
    "board_regex = ['board', 'chair', 'secretary', 'director$', 'trustee']\n",
    "board_antiregex = ['project', 'program', 'exec', 'deputy', 'academic',\n",
    "                   'engin', 'site', 'base', 'new york', 'sales', 'editorial',\n",
    "                  'progam',  # sic\n",
    "                  ]\n",
    "\n",
    "# founder\n",
    "founder_label = 'founder'\n",
    "founder_regex = ['founder']\n",
    "founder_antiregex = []\n",
    "\n",
    "# list of all labels\n",
    "labels = [ceo_label, cto_label, coo_label, cfo_label, bd_label,\n",
    "          pres_label, cmo_label, cdo_label, board_label, founder_label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for ceo found 241 rows at 105 orgs with 69 titles\n",
      "for cto found 129 rows at 34 orgs with 69 titles\n",
      "for coo found 74 rows at 37 orgs with 29 titles\n",
      "for cfo found 250 rows at 104 orgs with 66 titles\n",
      "for bd found 48 rows at 19 orgs with 33 titles\n",
      "for pres found 183 rows at 87 orgs with 45 titles\n",
      "for cmo found 13 rows at 9 orgs with 10 titles\n",
      "for cdo found 12 rows at 9 orgs with 8 titles\n",
      "for board found 2027 rows at 137 orgs with 131 titles\n",
      "for founder found 39 rows at 16 orgs with 22 titles\n"
     ]
    }
   ],
   "source": [
    "# set titles: 1 if match, 0 if not\n",
    "for label, regex_list, antiregex_list in [\n",
    "    (ceo_label, ceo_regex, ceo_antiregex),\n",
    "    (cto_label, cto_regex, cto_antiregex),\n",
    "    (coo_label, coo_regex, coo_antiregex),\n",
    "    (cfo_label, cfo_regex, cfo_antiregex),\n",
    "    (bd_label, bd_regex, bd_antiregex),\n",
    "    (pres_label, pres_regex, pres_antiregex),\n",
    "    (cmo_label, cmo_regex, cmo_antiregex),\n",
    "    (cdo_label, cdo_regex, cdo_antiregex),\n",
    "    (board_label, board_regex, board_antiregex),\n",
    "    (founder_label, founder_regex, founder_antiregex),\n",
    "]:\n",
    "    # column name is_label\n",
    "    colname = 'is_%s' % label\n",
    "\n",
    "    # default is not the title\n",
    "    df[colname] = 0\n",
    "    \n",
    "    # if matches good regex, it is the title\n",
    "    for regex in regex_list:\n",
    "        idx = df['title'].str.contains(regex, case=False)\n",
    "        df[colname].loc[idx] = 1\n",
    "        \n",
    "    # if matches anti-regex, override and make it not the title\n",
    "    for regex in antiregex_list:\n",
    "        idx = df['title'].str.contains(regex, case=False)\n",
    "        df[colname].loc[idx] = 0\n",
    "\n",
    "    # log\n",
    "    rows = df[df[colname] == 1]\n",
    "    print 'for', label, 'found', len(rows), 'rows at', len(rows['ein'].unique()), 'orgs with', len(rows['title'].unique()), 'titles'\n",
    "    # print rows['title'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    2678\n",
      "0     304\n",
      "2     166\n",
      "3       2\n",
      "Name: n_indiv_titles, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# number of titles held by this individual\n",
    "df['n_indiv_titles'] = df[['is_%s' % label for label in labels]].sum(axis=1)\n",
    "print df['n_indiv_titles'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for uncategorized found 304 rows at 71 orgs with 162 titles\n"
     ]
    }
   ],
   "source": [
    "# log uncategorized\n",
    "uncategorized_rows = df[df['n_indiv_titles'] == 0]\n",
    "print 'for uncategorized found', len(uncategorized_rows), 'rows at', len(uncategorized_rows['ein'].unique()), 'orgs with', len(uncategorized_rows['title'].unique()), 'titles'\n",
    "# print uncategorized_rows['title'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(u'ADOPT-A-CLASSROOM INC', u'2010')\n",
      "(u'ADOPT-A-CLASSROOM INC', u'2011')\n",
      "(u'ADOPT-A-CLASSROOM INC', u'2012')\n",
      "(u'ADOPT-A-CLASSROOM INC', u'2013')\n",
      "(u'ADOPT-A-CLASSROOM INC', u'2014')\n",
      "(u'Anjna Patient Education', u'2013')\n",
      "(u'BENEFICENT TECHNOLOGY INC DBA Benetech', u'2014')\n",
      "(u'BRACKETS FOR GOOD INC', u'2014')\n",
      "(u'Blue Planet Network', u'2011')\n",
      "(u'Blue Planet Network', u'2012')\n",
      "(u'CANCER COMMONS', u'2012')\n",
      "(u'CANCER COMMONS', u'2013')\n",
      "(u'CANCER COMMONS', u'2014')\n",
      "(u'CAREERVILLAGE INC', u'2012')\n",
      "(u'CAREERVILLAGE INC', u'2013')\n",
      "(u'CAREERVILLAGE INC', u'2014')\n",
      "(u'CASE COMMONS INC', u'2012')\n",
      "(u'CASE COMMONS INC', u'2013')\n",
      "(u'CASE COMMONS INC', u'2014')\n",
      "(u'CENTER FOR STUDENT OPPORTUNITY INC', u'2010')\n",
      "(u'CENTER FOR STUDENT OPPORTUNITY INC', u'2011')\n",
      "(u'CENTER FOR STUDENT OPPORTUNITY INC', u'2012')\n",
      "(u'CENTER FOR STUDENT OPPORTUNITY INC', u'2013')\n",
      "(u'CENTER FOR STUDENT OPPORTUNITY INC', u'2014')\n",
      "(u'CLASSROOM INC', u'2010')\n",
      "(u'CLASSROOM INC', u'2011')\n",
      "(u'CLASSROOM INC', u'2012')\n",
      "(u'CLASSROOM INC', u'2013')\n",
      "(u'CLASSROOM INC', u'2014')\n",
      "(u'CODE FOR AMERICA LABS INC', u'2010')\n",
      "(u'CODE FOR AMERICA LABS INC', u'2011')\n",
      "(u'CODE FOR AMERICA LABS INC', u'2012')\n",
      "(u'CODE FOR AMERICA LABS INC', u'2014')\n",
      "(u'CODE-TO-LEARN FOUNDATION', u'2013')\n",
      "(u'CODE-TO-LEARN FOUNDATION', u'2014')\n",
      "(u'CODEORG', u'2013')\n",
      "(u'CODEORG', u'2014')\n",
      "(u'COMMON SENSE MEDIA', u'2011')\n",
      "(u'COMMON SENSE MEDIA', u'2012')\n",
      "(u'COMMON SENSE MEDIA', u'2013')\n",
      "(u'COMMON SENSE MEDIA', u'2014')\n",
      "(u'CONNECT TO COMPETE INC', u'2014')\n",
      "(u'COWORKERORG', u'2013')\n",
      "(u'COWORKERORG', u'2014')\n",
      "(u'CRISIS TEXT LINE INC', u'2012')\n",
      "(u'CRISIS TEXT LINE INC', u'2013')\n",
      "(u'CRISIS TEXT LINE INC', u'2014')\n",
      "(u'D-REV DESIGN FOR THE OTHER 90', u'2010')\n",
      "(u'D-REV DESIGN FOR THE OTHER 90', u'2013')\n",
      "(u'D-REV DESIGN FOR THE OTHER 90', u'2014')\n",
      "(u'D-Rev Design for the Other 90', u'2012')\n",
      "(u'DEMOCRACY WORKS INC', u'2011')\n",
      "(u'DEMOCRACY WORKS INC', u'2012')\n",
      "(u'DEMOCRACY WORKS INC', u'2013')\n",
      "(u'DEMOCRACY WORKS INC', u'2014')\n",
      "(u'DESIGN THAT MATTERS INC', u'2011')\n",
      "(u'DESIGN THAT MATTERS INC', u'2012')\n",
      "(u'DESIGN THAT MATTERS INC', u'2013')\n",
      "(u'DESIGN THAT MATTERS INC', u'2014')\n",
      "(u'DIGITAL GREEN FOUNDATION', u'2010')\n",
      "(u'DIGITAL GREEN FOUNDATION', u'2011')\n",
      "(u'DIGITAL GREEN FOUNDATION', u'2012')\n",
      "(u'DIGITAL GREEN FOUNDATION', u'2013')\n",
      "(u'DIGITAL GREEN FOUNDATION', u'2014')\n",
      "(u'DO SOMETHING INC', u'2013')\n",
      "(u'DO SOMETHING INC', u'2014')\n",
      "(u'DONORSCHOOSEORG', u'2013')\n",
      "(u'DONORSCHOOSEORG', u'2014')\n",
      "(u'DTWO LTD', u'2011')\n",
      "(u'DTWO LTD', u'2012')\n",
      "(u'DTWO LTD', u'2013')\n",
      "(u'DTWO LTD', u'2014')\n",
      "(u'Do Something Inc', u'2011')\n",
      "(u'Do Something Inc', u'2012')\n",
      "(u'E-THE PEOPLE', u'2014')\n",
      "(u'EDNOVO', u'2011')\n",
      "(u'EDNOVO', u'2012')\n",
      "(u'EDNOVO', u'2013')\n",
      "(u'EDX INC', u'2013')\n",
      "(u'Elephant Action League', u'2014')\n",
      "(u'FAIR TRADE FUND INC', u'2012')\n",
      "(u'FAIR TRADE FUND INC', u'2013')\n",
      "(u'FAMILIES EMPOWERED', u'2010')\n",
      "(u'FAMILIES EMPOWERED', u'2011')\n",
      "(u'FAMILIES EMPOWERED', u'2012')\n",
      "(u'FAMILIES EMPOWERED', u'2013')\n",
      "(u'FAMILIES EMPOWERED', u'2014')\n",
      "(u'GIVE DIRECT INC', u'2011')\n",
      "(u'GIVEWELL COMMUNITY FOUNDATION INC', u'2014')\n",
      "(u'GLOBAL LIVES PROJECT INC', u'2011')\n",
      "(u'GLOBAL LIVES PROJECT INC', u'2013')\n",
      "(u'GLOBAL LIVES PROJECT INC', u'2014')\n",
      "(u'GOOD WORLD SOLUTIONS INC', u'2012')\n",
      "(u'GOOD WORLD SOLUTIONS INC', u'2013')\n",
      "(u'GOOD WORLD SOLUTIONS INC', u'2014')\n",
      "(u'Great NonProfits', u'2011')\n",
      "(u'Great NonProfits', u'2012')\n",
      "(u'Great NonProfits', u'2013')\n",
      "(u'Great NonProfits', u'2014')\n",
      "(u'HARMONY INSTITUTE INC', u'2010')\n",
      "(u'HARMONY INSTITUTE INC', u'2011')\n",
      "(u'HARMONY INSTITUTE INC', u'2012')\n",
      "(u'HARMONY INSTITUTE INC', u'2013')\n",
      "(u'HARMONY INSTITUTE INC', u'2014')\n",
      "(u'HOLLABACK INC', u'2011')\n",
      "(u'HOLLABACK INC', u'2012')\n",
      "(u'HOLLABACK INC', u'2013')\n",
      "(u'HOLLABACK INC', u'2014')\n",
      "(u'HOPING HEARTS FOUNDATION INC', u'2012')\n",
      "(u'HOPING HEARTS FOUNDATION INC', u'2013')\n",
      "(u'HOPING HEARTS FOUNDATION INC', u'2014')\n",
      "(u'HUMANITARIAN OPENSTREETMAP TEAM UNITED STATES INC', u'2012')\n",
      "(u'HopeLab Foundation Inc', u'2010')\n",
      "(u'HopeLab Foundation Inc', u'2011')\n",
      "(u'HopeLab Foundation Inc', u'2012')\n",
      "(u'HopeLab Foundation Inc', u'2013')\n",
      "(u'HopeLab Foundation Inc', u'2014')\n",
      "(u'IMAGING THE WORLD CORP', u'2013')\n",
      "(u'IMAGING THE WORLD CORP', u'2014')\n",
      "(u'IMAGING THE WORLD CORPORATION', u'2010')\n",
      "(u'IMAGING THE WORLD CORPORATION', u'2011')\n",
      "(u'IMAGING THE WORLD CORPORATION', u'2012')\n",
      "(u'IN OUR BACKYARDS INC', u'2010')\n",
      "(u'IN OUR BACKYARDS INC', u'2011')\n",
      "(u'IN OUR BACKYARDS INC', u'2012')\n",
      "(u'IN OUR BACKYARDS INC', u'2013')\n",
      "(u'IN OUR BACKYARDS INC', u'2014')\n",
      "(u'INTERNET SEXUALITY INFORMATION SERVICES', u'2014')\n",
      "(u'INTERNEWS NETWORK INC', u'2012')\n",
      "(u'INTERNEWS NETWORK INC', u'2013')\n",
      "(u'INTERNEWS NETWORK INC', u'2014')\n",
      "(u'Impact Network International Inc', u'2014')\n",
      "(u'InSTEDD', u'2011')\n",
      "(u'InSTEDD', u'2012')\n",
      "(u'InSTEDD', u'2013')\n",
      "(u'InSTEDD', u'2014')\n",
      "(u'International Bridges to Justice Inc', u'2014')\n",
      "(u'KANGU INC', u'2013')\n",
      "(u'KANGU INC', u'2014')\n",
      "(u'KHAN ACADEMY INC', u'2012')\n",
      "(u'KHAN ACADEMY INC', u'2013')\n",
      "(u'KHAN ACADEMY INC', u'2014')\n",
      "(u'LEARN FRESH EDUCATION CO', u'2014')\n",
      "(u'LIFEBOX FOUNDATION INC', u'2013')\n",
      "(u'LITERACY BRIDGE', u'2010')\n",
      "(u'LITERACY BRIDGE', u'2011')\n",
      "(u'LITERACY BRIDGE', u'2012')\n",
      "(u'LITERACY LAB', u'2011')\n",
      "(u'Lifebox Foundation Inc', u'2014')\n",
      "(u'Literacy Bridge', u'2013')\n",
      "(u'Literacy Bridge', u'2014')\n",
      "(u'MEDIC MOBILE INC', u'2012')\n",
      "(u'MEDIC MOBILE INC', u'2013')\n",
      "(u'MEDIC MOBILE INC', u'2014')\n",
      "(u'MEN OF COURAGE FOUNDATION DBA UNCOMMEN', u'2014')\n",
      "(u'MIND RESEARCH INSTITUTE', u'2011')\n",
      "(u'MIND RESEARCH INSTITUTE', u'2012')\n",
      "(u'MIND RESEARCH INSTITUTE', u'2013')\n",
      "(u'MIND Research Institute', u'2014')\n",
      "(u'MOZILLA FOUNDATION', u'2014')\n",
      "(u'Moneythink', u'2012')\n",
      "(u'Moneythink', u'2013')\n",
      "(u'Moneythink', u'2014')\n",
      "(u'NATIONAL INSTITUTE ON MONEY IN STATE POLITICS', u'2012')\n",
      "(u'NATIONAL INSTITUTE ON MONEY IN STATE POLITICS', u'2013')\n",
      "(u'NATIONAL INSTITUTE ON MONEY IN STATE POLITICS', u'2014')\n",
      "(u'NEW CLASSROOMS INNOVATION PARTNERSINC', u'2011')\n",
      "(u'NEW CLASSROOMS INNOVATION PARTNERSINC', u'2012')\n",
      "(u'NEW CLASSROOMS INNOVATION PARTNERSINC', u'2013')\n",
      "(u'NEW CLASSROOMS INNOVATION PARTNERSINC', u'2014')\n",
      "(u'NEWBORN FOUNDATION', u'2011')\n",
      "(u'NEWBORN FOUNDATION', u'2012')\n",
      "(u'NEXLEAF ANALYTICS', u'2011')\n",
      "(u'NEXLEAF ANALYTICS', u'2012')\n",
      "(u'NEXLEAF ANALYTICS', u'2013')\n",
      "(u'NEXLEAF ANALYTICS', u'2014')\n",
      "(u'NPOWER INC', u'2014')\n",
      "(u'NPOWERNY INC', u'2012')\n",
      "(u'National Institute on Money in State', u'2010')\n",
      "(u'National Institute on Money in State', u'2011')\n",
      "(u'OCEARCH', u'2012')\n",
      "(u'OCEARCH', u'2013')\n",
      "(u'OCEARCH', u'2014')\n",
      "(u'ONE LAPTOP PER CHILD ASSOCIATION INC', u'2010')\n",
      "(u'ONE LAPTOP PER CHILD ASSOCIATION INC', u'2011')\n",
      "(u'ONE LAPTOP PER CHILD ASSOCIATION INC', u'2012')\n",
      "(u'OPEN MEDIA FOUNDATION', u'2010')\n",
      "(u'OPEN MEDIA FOUNDATION', u'2011')\n",
      "(u'OPEN MEDIA FOUNDATION', u'2012')\n",
      "(u'OPEN MEDIA FOUNDATION', u'2013')\n",
      "(u'OPEN MEDIA FOUNDATION', u'2014')\n",
      "(u'OPERATION ASHA NFP', u'2013')\n",
      "(u'OPERATION ASHA NFP', u'2014')\n",
      "(u'PARTICIPATORY CULTURE FOUNDATION INC', u'2010')\n",
      "(u'PARTICIPATORY CULTURE FOUNDATION INC', u'2011')\n",
      "(u'PARTICIPATORY CULTURE FOUNDATION INC', u'2012')\n",
      "(u'PARTICIPATORY CULTURE FOUNDATION INC', u'2013')\n",
      "(u'PARTICIPATORY CULTURE FOUNDATION INC', u'2014')\n",
      "(u'PARTICIPATORY POLITICS FOUNDATION', u'2010')\n",
      "(u'PARTICIPATORY POLITICS FOUNDATION', u'2011')\n",
      "(u'PARTICIPATORY POLITICS FOUNDATION', u'2012')\n",
      "(u'PARTICIPATORY POLITICS FOUNDATION', u'2013')\n",
      "(u'PARTICIPATORY POLITICS FOUNDATION', u'2014')\n",
      "(u'PLANETWORK NGO INC', u'2015')\n",
      "(u'PULSEPOINT FOUNDATION', u'2012')\n",
      "(u'PULSEPOINT FOUNDATION', u'2013')\n",
      "(u'PULSEPOINT FOUNDATION', u'2014')\n",
      "(u'Planetwork NGO Inc', u'2014')\n",
      "(u'REALLOCATE INC', u'2012')\n",
      "(u'REALLOCATE INC', u'2013')\n",
      "(u'REALLOCATE INC', u'2014')\n",
      "(u'REASONING MIND INC', u'2012')\n",
      "(u'REASONING MIND INC', u'2013')\n",
      "(u'REASONING MIND INC', u'2014')\n",
      "(u'ReadworksInc', u'2012')\n",
      "(u'ReadworksInc', u'2013')\n",
      "(u'SAMASOURCE INC', u'2010')\n",
      "(u'SAMASOURCE INC', u'2011')\n",
      "(u'SAMASOURCE INC', u'2013')\n",
      "(u'SCIENCE BUDDIES', u'2013')\n",
      "(u'SCIENCE BUDDIES', u'2014')\n",
      "(u'SKYTRUTH', u'2011')\n",
      "(u'Single Stop USA Inc', u'2011')\n",
      "(u'Single Stop USA Inc', u'2012')\n",
      "(u'Single Stop USA Inc', u'2013')\n",
      "(u'Single Stop USA Inc', u'2014')\n",
      "(u'SkyTruth', u'2010')\n",
      "(u'SkyTruth', u'2012')\n",
      "(u'SkyTruth', u'2013')\n",
      "(u'SkyTruth', u'2014')\n",
      "(u'TEACHAIDS', u'2014')\n",
      "(u'TECHSOUP GLOBAL', u'2009')\n",
      "(u'TECHSOUP GLOBAL', u'2010')\n",
      "(u'TECHSOUP GLOBAL', u'2011')\n",
      "(u'TECHSOUP GLOBAL', u'2012')\n",
      "(u'TECHSOUP GLOBAL', u'2013')\n",
      "(u'TECHSOUP GLOBAL', u'2014')\n",
      "(u'THE 1947 PARTITION ARCHIVE', u'2012')\n",
      "(u'THE 1947 PARTITION ARCHIVE', u'2013')\n",
      "(u'THE 1947 PARTITION ARCHIVE', u'2014')\n",
      "(u'THE CENTER TO PROMOTE HEALTHCARE ACCESS INC', u'2011')\n",
      "(u'THE CENTER TO PROMOTE HEALTHCARE ACCESS INC', u'2012')\n",
      "(u'THE CENTER TO PROMOTE HEALTHCARE ACCESS INC', u'2013')\n",
      "(u'THE CENTER TO PROMOTE HEALTHCARE ACCESS INC', u'2014')\n",
      "(u'THE FREECYCLE NETWORK', u'2009')\n",
      "(u'THE FREECYCLE NETWORK', u'2011')\n",
      "(u'THE FREECYCLE NETWORK', u'2012')\n",
      "(u'THE FREECYCLE NETWORK', u'2014')\n",
      "(u'THE GET SCHOOLED FOUNDATION', u'2012')\n",
      "(u'THE GET SCHOOLED FOUNDATION', u'2013')\n",
      "(u'THE GET SCHOOLED FOUNDATION', u'2014')\n",
      "(u'THE KIVA FOUNDATION', u'2010')\n",
      "(u'THE KIVA FOUNDATION', u'2011')\n",
      "(u'THE KIVA FOUNDATION', u'2012')\n",
      "(u'THE KIVA FOUNDATION', u'2013')\n",
      "(u'THE KIVA FOUNDATION', u'2014')\n",
      "(u'THE LITERACY LAB', u'2012')\n",
      "(u'THE LITERACY LAB', u'2013')\n",
      "(u'THE LITERACY LAB', u'2014')\n",
      "(u'THE OPENGOV FOUNDATION', u'2013')\n",
      "(u'THE OPENGOV FOUNDATION', u'2014')\n",
      "(u'THE STELLAR FOUNDATION', u'2012')\n",
      "(u'THE STELLAR FOUNDATION', u'2013')\n",
      "(u'THE STELLAR FOUNDATION', u'2014')\n",
      "(u'TIDEPOOL PROJECT', u'2013')\n",
      "(u'TIDEPOOL PROJECT', u'2014')\n",
      "(u'TRANSPARENCY TOOLKIT INC', u'2014')\n",
      "(u'The Center to Promote Healthcare Access Inc', u'2009')\n",
      "(u'The Lunchbox Fund', u'2012')\n",
      "(u'The Lunchbox Fund', u'2013')\n",
      "(u'The Lunchbox Fund', u'2014')\n",
      "(u'The MakeSense Foundation', u'2014')\n",
      "(u'UNIVERSALGIVING', u'2011')\n",
      "(u'UNIVERSALGIVING', u'2012')\n",
      "(u'UNIVERSALGIVING', u'2013')\n",
      "(u'UNIVERSALGIVING', u'2014')\n",
      "(u'Video Volunteers', u'2009')\n",
      "(u'Video Volunteers', u'2010')\n",
      "(u'Video Volunteers', u'2011')\n",
      "(u'Video Volunteers', u'2013')\n",
      "(u'Video Volunteers', u'2014')\n",
      "(u'WATSI INC', u'2012')\n",
      "(u'WATSI INC', u'2013')\n",
      "(u'WISHBONEORG', u'2011')\n",
      "(u'WISHBONEORG', u'2012')\n",
      "(u'WISHBONEORG', u'2013')\n",
      "(u'WISHBONEORG', u'2014')\n",
      "(u'WITNESS INC', u'2014')\n",
      "(u'WORLDREADERORG', u'2011')\n",
      "(u'WORLDREADERORG', u'2013')\n",
      "(u'WORLDREADERORG', u'2014')\n",
      "(u'Wikimedia Foundation Inc', u'2009')\n",
      "(u'Wikimedia Foundation Inc', u'2010')\n",
      "(u'Wikimedia Foundation Inc', u'2011')\n",
      "(u'Wikimedia Foundation Inc', u'2012')\n",
      "(u'Wikimedia Foundation Inc', u'2013')\n",
      "(u'Wikimedia Foundation Inc', u'2014')\n",
      "(u'ZEARN INC', u'2014')\n",
      "(u'ZIDISHA INC', u'2012')\n",
      "(u'ZIDISHA INC', u'2013')\n",
      "(u'ZIDISHA INC', u'2014')\n",
      "(u'edX Inc', u'2014')\n",
      "(u'iCivics Inc', u'2009')\n",
      "(u'iCivics Inc', u'2010')\n",
      "(u'iCivics Inc', u'2011')\n",
      "(u'iCivics Inc', u'2012')\n",
      "(u'iCivics Inc', u'2013')\n",
      "(u'iCivics Inc', u'2014')\n",
      "(u'myAgro Farms', u'2013')\n",
      "(u'myAgro Farms', u'2014')\n",
      "(u'weTHRIVE INC', u'2013')\n",
      "(u'weTHRIVE INC', u'2014')\n"
     ]
    }
   ],
   "source": [
    "# set up features for what's happening across the org in that year\n",
    "for group_tuple, group in df.groupby(['org_name', 'tax_year']):\n",
    "    print group_tuple\n",
    "    for label in labels:\n",
    "        # number of people with this title\n",
    "        n_title_in_org_year = group['is_%s' % label].sum()\n",
    "        \n",
    "        # set number on df\n",
    "        df.loc[group.index, 'n_%s_in_org_year' % label] = n_title_in_org_year\n",
    "\n",
    "        # set boolean for whether title is present (0/1)\n",
    "        df.loc[group.index, 'has_%s_in_org_year' % label] = 1 if n_title_in_org_year > 0 else 0\n",
    "        \n",
    "        # total compensation for people with this title\n",
    "        total_comp_for_title = group[group['is_%s' % label] == 1]['comp'].sum()\n",
    "        \n",
    "        # set compensation on df\n",
    "        df.loc[group.index, 'total_comp_%s_in_org_year' % label] = total_comp_for_title\n",
    "\n",
    "        # set boolean for whether title is compensated (0/1)\n",
    "        df.loc[group.index, 'has_comp_%s_in_org_year' % label] = 1 if total_comp_for_title > 0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# other features\n",
    "df['org_age'] = pd.to_numeric(df['tax_year'], errors='coerce') - pd.to_numeric(df['year_formed'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "have features ['org_name' 'city' 'state' 'ein' 'url' 'tax_year' 'year_formed'\n",
      " 'submission_time' 'n_employees' 'total_revenue' 'title' 'comp' 'is_ceo'\n",
      " 'is_cto' 'is_coo' 'is_cfo' 'is_bd' 'is_pres' 'is_cmo' 'is_cdo' 'is_board'\n",
      " 'is_founder' 'n_indiv_titles' 'n_ceo_in_org_year' 'has_ceo_in_org_year'\n",
      " 'total_comp_ceo_in_org_year' 'has_comp_ceo_in_org_year'\n",
      " 'n_cto_in_org_year' 'has_cto_in_org_year' 'total_comp_cto_in_org_year'\n",
      " 'has_comp_cto_in_org_year' 'n_coo_in_org_year' 'has_coo_in_org_year'\n",
      " 'total_comp_coo_in_org_year' 'has_comp_coo_in_org_year'\n",
      " 'n_cfo_in_org_year' 'has_cfo_in_org_year' 'total_comp_cfo_in_org_year'\n",
      " 'has_comp_cfo_in_org_year' 'n_bd_in_org_year' 'has_bd_in_org_year'\n",
      " 'total_comp_bd_in_org_year' 'has_comp_bd_in_org_year' 'n_pres_in_org_year'\n",
      " 'has_pres_in_org_year' 'total_comp_pres_in_org_year'\n",
      " 'has_comp_pres_in_org_year' 'n_cmo_in_org_year' 'has_cmo_in_org_year'\n",
      " 'total_comp_cmo_in_org_year' 'has_comp_cmo_in_org_year'\n",
      " 'n_cdo_in_org_year' 'has_cdo_in_org_year' 'total_comp_cdo_in_org_year'\n",
      " 'has_comp_cdo_in_org_year' 'n_board_in_org_year' 'has_board_in_org_year'\n",
      " 'total_comp_board_in_org_year' 'has_comp_board_in_org_year'\n",
      " 'n_founder_in_org_year' 'has_founder_in_org_year'\n",
      " 'total_comp_founder_in_org_year' 'has_comp_founder_in_org_year' 'org_age'\n",
      " 'city_int' 'state_int' u'in_ARCATA' u'in_BARRINGTON' u'in_BELLVALE'\n",
      " u'in_BERKELEY' u'in_BETHESDA' u'in_BETHSEDA' u'in_BOSTON' u'in_BROOKLYN'\n",
      " u'in_CAMBRIDGE' u'in_CARMEL' u'in_CHARLOTTE' u'in_CHICAGO' u'in_DANVERS'\n",
      " u'in_DENVER' u'in_ENCINO' u'in_HELENA' u'in_HOUSTON' u'in_HYANNIS'\n",
      " u'in_IRVINE' u'in_JACKSONVILLE' u'in_LAKELAND' u'in_LOS ANGELES'\n",
      " u'in_MIAMI' u'in_MINNEAPOLIS' u'in_MOUNTAIN VIEW' u'in_NEW YORK'\n",
      " u'in_OAKLAND' u'in_PALO ALTO' u'in_PARK CITY' u'in_PLEASANTON'\n",
      " u'in_REDWOOD CITY' u'in_SALEM' u'in_SAN FRANCISCO' u'in_SANTA ANA'\n",
      " u'in_SEATTLE' u'in_SHEPHERDSTOWN' u'in_STANFORD' u'in_STERLING'\n",
      " u'in_SUNNYVALE' u'in_TUCSON' u'in_WALTHAM' u'in_WARWICK' u'in_WASHINGTON'\n",
      " u'in_WAYNESVILLE' u'in_WEST SACRAMENTO' u'in_ZIONSVILLE' u'in_AZ' u'in_CA'\n",
      " u'in_CO' u'in_DC' u'in_FL' u'in_IL' u'in_IN' u'in_MA' u'in_MD' u'in_MN'\n",
      " u'in_MT' u'in_NC' u'in_NY' u'in_OH' u'in_RI' u'in_TX' u'in_UT' u'in_VA'\n",
      " u'in_VT' u'in_WA' u'in_WV']\n"
     ]
    }
   ],
   "source": [
    "# convert categorical features (city and state) to boolean\n",
    "from sklearn import preprocessing\n",
    "\n",
    "# integer code for cities\n",
    "df['city'] = df['city'].str.upper()\n",
    "cities = sorted(df['city'].unique())\n",
    "df['city_int'] = df['city'].map(lambda x: cities.index(x))\n",
    "\n",
    "# integer code for states\n",
    "states = sorted(df['state'].unique())\n",
    "df['state_int'] = df['state'].map(lambda x: states.index(x))\n",
    "\n",
    "# perform one-hot enconding\n",
    "enc = preprocessing.OneHotEncoder()\n",
    "enc.fit(df[['city_int', 'state_int']])\n",
    "transformed = enc.transform(df[['city_int', 'state_int']]).toarray()\n",
    "\n",
    "# add features to df\n",
    "header = ['in_%s' % city for city in cities] + ['in_%s' % state for state in states]\n",
    "transformed_df = pd.DataFrame(transformed, columns=header)\n",
    "df = pd.concat([df, transformed_df], axis=1)\n",
    "print 'have features', df.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output\n",
    "df.to_csv('output/comp_data_features.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}