migrate to em-http-request, to avoid timeouts.
parent
f6911da8c6
commit
a1d4c856ab
|
@ -1,4 +1,4 @@
|
||||||
require 'net/http'
|
require 'em-net-http'
|
||||||
require 'json'
|
require 'json'
|
||||||
require 'time'
|
require 'time'
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
require 'net/http'
|
require 'em-net-http'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
|
||||||
Client.register_command("cycki") { |args|
|
Client.register_command("cycki") { |args|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
require 'net/http'
|
require 'em-net-http'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
|
||||||
Client.register_command("bug") { |args|
|
Client.register_command("bug") { |args|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
require 'net/http'
|
require 'em-net-http'
|
||||||
require 'uri'
|
require 'uri'
|
||||||
|
|
||||||
Client.register_command("kotki") { |args|
|
Client.register_command("kotki") { |args|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
require 'net/http'
|
require 'em-net-http'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
|
||||||
Client.register_command("sjp") { |args|
|
Client.register_command("sjp") { |args|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
require 'open-uri'
|
require 'em-http-request'
|
||||||
require 'digest/md5'
|
require 'digest/md5'
|
||||||
require 'data_mapper'
|
require 'data_mapper'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
@ -10,7 +10,7 @@ DataMapper.setup(:default, Config[:scrape][:db])
|
||||||
module Scrape
|
module Scrape
|
||||||
def self.download(uri)
|
def self.download(uri)
|
||||||
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
|
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
|
||||||
content = open(uri).read
|
content = Net::HTTP.get(uri)
|
||||||
md5 = Digest::MD5.hexdigest(content)
|
md5 = Digest::MD5.hexdigest(content)
|
||||||
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
||||||
file.write(content)
|
file.write(content)
|
||||||
|
@ -45,7 +45,7 @@ Client.register_trigger("PRIVMSG") do |msg|
|
||||||
url = Digest::MD5.hexdigest(word)
|
url = Digest::MD5.hexdigest(word)
|
||||||
end
|
end
|
||||||
|
|
||||||
title = Nokogiri::HTML(open(word).read(10240)).title()
|
title = Nokogiri::HTML(Net::HTTP.get(word)).title()
|
||||||
title = title.gsub(/[\s]+/, " ").strip if not title.nil?
|
title = title.gsub(/[\s]+/, " ").strip if not title.nil?
|
||||||
|
|
||||||
Scrape::Link.create(
|
Scrape::Link.create(
|
||||||
|
|
Loading…
Reference in New Issue