first commit

Author: steveodom

Gemfile

@@ -0,0 +1,35 @@
+source "https://rubygems.org"
+ruby '2.1.1'
+
+gem 'sinatra'
+gem 'rack' #for static site
+gem 'rack-contrib'
+
+gem 'pry'
+# gem 'pry-debugger' #add step, next
+gem 'airbrake'
+gem 'sucker_punch'
+
+gem 'mongoid',  "~> 3.1.2"
+gem 'bson_ext'
+
+gem 'redis'
+# gem 'nokogiri'
+
+# for loading the .env file locally
+gem 'dotenv', :groups => [:development, :test]
+
+# gem "geocoder"
+gem 'faraday'
+gem 'faraday_middleware'
+gem 'simple_oauth'
+
+# needed because open-uri won't allow to redirect from http to https and vice versa. needed to get base url
+gem 'open_uri_redirections'
+
+group :test do
+  # gem 'spinach'
+  gem "rack-test"
+  gem "factory_girl"
+  gem 'rspec'
+end

Gemfile.lock

@@ -0,0 +1,92 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (3.2.15)
+      activesupport (= 3.2.15)
+      builder (~> 3.0.0)
+    activesupport (3.2.15)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    airbrake (4.1.0)
+      builder
+      multi_json
+    bson (1.9.2)
+    bson_ext (1.9.2)
+      bson (~> 1.9.2)
+    builder (3.0.4)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    dotenv (0.9.0)
+    factory_girl (4.3.0)
+      activesupport (>= 3.0.0)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    faraday_middleware (0.9.0)
+      faraday (>= 0.7.4, < 0.9)
+    i18n (0.6.5)
+    method_source (0.8.2)
+    mongoid (3.1.5)
+      activemodel (~> 3.2)
+      moped (~> 1.4)
+      origin (~> 1.0)
+      tzinfo (~> 0.3.29)
+    moped (1.5.1)
+    multi_json (1.8.2)
+    multipart-post (1.2.0)
+    open_uri_redirections (0.1.4)
+    origin (1.1.0)
+    pry (0.9.12.3)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rack-contrib (1.1.0)
+      rack (>= 0.9.1)
+    rack-protection (1.5.1)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    redis (3.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    simple_oauth (0.2.0)
+    sinatra (1.4.4)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.7)
+    sucker_punch (1.1)
+      celluloid (~> 0.15.2)
+    tilt (1.4.1)
+    timers (1.1.0)
+    tzinfo (0.3.38)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  airbrake
+  bson_ext
+  dotenv
+  factory_girl
+  faraday
+  faraday_middleware
+  mongoid (~> 3.1.2)
+  open_uri_redirections
+  pry
+  rack
+  rack-contrib
+  rack-test
+  redis
+  rspec
+  simple_oauth
+  sinatra
+  sucker_punch

Procfile

@@ -0,0 +1 @@
+web: bundle exec rackup config.ru -p $PORT

README.md

@@ -0,0 +1,4 @@
+Template: add description of what this app does
+
+Use as console:
+bundle exec ruby console.rb

app.rb

@@ -0,0 +1,6 @@
+require 'sinatra'
+require './config/initializer'
+
+before do
+  content_type :json
+end

config.ru

@@ -0,0 +1,7 @@
+require 'rubygems'
+Bundler.require
+
+require './app'
+
+run Rack::URLMap.new \
+  "/"       => Sinatra::Application

config/initializer.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'pry'
+require 'json'
+require 'faraday'
+
+# for Twitter.base_uri
+require 'open-uri'
+require 'open_uri_redirections'
+
+# # require 'newrelic_rpm'
+
+if ENV['RACK_ENV'] != "production"
+	require 'dotenv'
+	Dotenv.load
+end
+
+Dir['./config/initializers/*.rb'].each{ |f| require f }
+Dir['./models/*.rb'].each{ |f| require f }
+Dir['./lib/*.rb'].each{ |f| require f }

config/initializers/airbrake.rb

@@ -0,0 +1,6 @@
+require 'airbrake'
+
+Airbrake.configure do |config|
+  config.api_key = ENV['AIRBRAKE_KEY']
+  config.async = true
+end

config/initializers/mongo.rb

@@ -0,0 +1,4 @@
+require 'mongoid'
+# mongolab password is same as aws account key
+
+Mongoid.load!('./config/mongoid.yml')

config/initializers/redis.rb

@@ -0,0 +1,4 @@
+require 'redis'
+
+# use a different database if only this app will need it. use 0 where everything will be shared.
+REDIS = Redis.new(:host => ENV['REDIS_HOST'], :port => ENV['REDIS_PORT'], :password => ENV['REDIS_AUTH'], :db => 0)

config/mongoid.yml

@@ -0,0 +1,16 @@
+development:
+  sessions:
+    default:
+      uri: <%= ENV['MONGOHQ_URL'] %>
+
+production:
+  sessions:
+    default:
+      uri: <%= ENV['MONGOHQ_URL'] %>
+
+test:
+  sessions:
+    default:
+      database: jetsam-test
+      hosts:
+        - localhost:27017

console.rb

@@ -0,0 +1,3 @@
+require './config/initializer.rb'
+
+binding.pry

lib/redis_key.rb

@@ -0,0 +1,30 @@
+class RedisKey
+
+	attr_accessor :registry
+
+	def initialize(prefix)
+		@prefix = prefix
+		@registry = {
+			"default" => {
+				:name 		=> "example",
+				:data_type 	=> "string",
+				:desc 		=> "stores example data"
+			} 
+		}
+	end
+
+	def get(name, arg = {})
+		if @registry[name]
+			str = "#{@prefix}:#{name}"
+
+			if !arg.empty?
+				str += ":#{arg}"
+			end
+			return str
+		else
+			raise NotFound
+		end
+	end
+
+	class NotFound < StandardError; end
+end

models/link.rb

@@ -0,0 +1,67 @@
+class Link
+
+	def self.ignore_list
+		keyword_file = File.join(Dir.pwd, "lib/ignore.yml")
+		YAML::load( File.open( keyword_file ) )
+	end
+
+	def self.aggregate_across_list(list_owner, list_name)
+		ignore = ignore_list
+		domains = {}
+		puts ignore.inspect
+		members = Twitter.list_members(list_owner, list_name)
+		members.each do |screen_name, twid|
+			Link.new(screen_name, {:ignore => ignore }).from_twitter(domains)
+		end
+		domains
+	end
+
+	def initialize(screen_name, options = {})
+		@screen_name = screen_name
+		if options[:ignore]
+			@ignore = options[:ignore]
+		end
+	end
+
+	def from_twitter(domains = {})
+		puts "screen_name: #{@screen_name}"
+		res = Twitter.timeline(@screen_name)
+		res.each do |tweet|
+			if !tweet['entities']['urls'].empty?
+				tweet['entities']['urls'].each do |obj|
+					begin
+						shared = obj['expanded_url']
+						link = Link.base_uri(shared)
+						url = URI.parse(link)
+						domain = PublicSuffix.parse(url.host).domain
+						puts domain
+						next if @ignore && @ignore.include?(domain)
+
+						if domains[domain]
+							domains[domain] += 1
+						else
+							domains[domain] = 1
+						end
+					rescue => e
+						puts "LINK problem: #{e.message}"
+					end
+				end
+			end
+		end
+		puts "__________________________________"
+		domains
+	end
+
+	def self.base_uri(url)
+		begin
+			Timeout.timeout(3) do
+				open(url,:allow_redirections => :safe) do |res|
+					res.base_uri.to_s
+				end
+			end
+		rescue => e
+			puts "TIMEOUT for #{url}"
+			url
+		end
+	end
+end