migrate to em-http-request, to avoid timeouts.
This commit is contained in:
parent
f6911da8c6
commit
a1d4c856ab
6 changed files with 8 additions and 8 deletions
|
@ -1,4 +1,4 @@
|
|||
require 'net/http'
|
||||
require 'em-net-http'
|
||||
require 'json'
|
||||
require 'time'
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
require 'net/http'
|
||||
require 'em-net-http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("cycki") { |args|
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
require 'net/http'
|
||||
require 'em-net-http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("bug") { |args|
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
require 'net/http'
|
||||
require 'em-net-http'
|
||||
require 'uri'
|
||||
|
||||
Client.register_command("kotki") { |args|
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
require 'net/http'
|
||||
require 'em-net-http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("sjp") { |args|
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
require 'open-uri'
|
||||
require 'em-http-request'
|
||||
require 'digest/md5'
|
||||
require 'data_mapper'
|
||||
require 'nokogiri'
|
||||
|
@ -10,7 +10,7 @@ DataMapper.setup(:default, Config[:scrape][:db])
|
|||
module Scrape
|
||||
def self.download(uri)
|
||||
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
|
||||
content = open(uri).read
|
||||
content = Net::HTTP.get(uri)
|
||||
md5 = Digest::MD5.hexdigest(content)
|
||||
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
||||
file.write(content)
|
||||
|
@ -45,7 +45,7 @@ Client.register_trigger("PRIVMSG") do |msg|
|
|||
url = Digest::MD5.hexdigest(word)
|
||||
end
|
||||
|
||||
title = Nokogiri::HTML(open(word).read(10240)).title()
|
||||
title = Nokogiri::HTML(Net::HTTP.get(word)).title()
|
||||
title = title.gsub(/[\s]+/, " ").strip if not title.nil?
|
||||
|
||||
Scrape::Link.create(
|
||||
|
|
Loading…
Reference in a new issue