migrate to em-http-request, to avoid timeouts.

master
Robert Gerus 2015-04-14 16:31:51 +02:00
parent f6911da8c6
commit a1d4c856ab
6 changed files with 8 additions and 8 deletions

View File

@ -1,4 +1,4 @@
require 'net/http'
require 'em-net-http'
require 'json'
require 'time'

View File

@ -1,4 +1,4 @@
require 'net/http'
require 'em-net-http'
require 'nokogiri'
Client.register_command("cycki") { |args|

View File

@ -1,4 +1,4 @@
require 'net/http'
require 'em-net-http'
require 'nokogiri'
Client.register_command("bug") { |args|

View File

@ -1,4 +1,4 @@
require 'net/http'
require 'em-net-http'
require 'uri'
Client.register_command("kotki") { |args|

View File

@ -1,4 +1,4 @@
require 'net/http'
require 'em-net-http'
require 'nokogiri'
Client.register_command("sjp") { |args|

View File

@ -1,4 +1,4 @@
require 'open-uri'
require 'em-http-request'
require 'digest/md5'
require 'data_mapper'
require 'nokogiri'
@ -10,7 +10,7 @@ DataMapper.setup(:default, Config[:scrape][:db])
module Scrape
def self.download(uri)
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
content = open(uri).read
content = Net::HTTP.get(uri)
md5 = Digest::MD5.hexdigest(content)
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
file.write(content)
@ -45,7 +45,7 @@ Client.register_trigger("PRIVMSG") do |msg|
url = Digest::MD5.hexdigest(word)
end
title = Nokogiri::HTML(open(word).read(10240)).title()
title = Nokogiri::HTML(Net::HTTP.get(word)).title()
title = title.gsub(/[\s]+/, " ").strip if not title.nil?
Scrape::Link.create(