Compare commits
3 Commits
configurat
...
master
Author | SHA1 | Date |
---|---|---|
![]() |
b2c2eec12d | |
![]() |
a8eb42a080 | |
![]() |
a1d4c856ab |
|
@ -1,4 +1,3 @@
|
|||
require 'net/http'
|
||||
require 'json'
|
||||
require 'time'
|
||||
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'net/http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("cycki") { |args|
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'net/http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("bug") { |args|
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'net/http'
|
||||
require 'uri'
|
||||
|
||||
Client.register_command("kotki") { |args|
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'net/http'
|
||||
require 'nokogiri'
|
||||
|
||||
Client.register_command("sjp") { |args|
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'open-uri'
|
||||
require 'digest/md5'
|
||||
require 'data_mapper'
|
||||
require 'nokogiri'
|
||||
|
@ -10,7 +9,7 @@ DataMapper.setup(:default, Config[:scrape][:db])
|
|||
module Scrape
|
||||
def self.download(uri)
|
||||
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
|
||||
content = open(uri).read
|
||||
content = Net::HTTP.get(uri)
|
||||
md5 = Digest::MD5.hexdigest(content)
|
||||
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
||||
file.write(content)
|
||||
|
@ -45,7 +44,7 @@ Client.register_trigger("PRIVMSG") do |msg|
|
|||
url = Digest::MD5.hexdigest(word)
|
||||
end
|
||||
|
||||
title = Nokogiri::HTML(open(word).read(10240)).title()
|
||||
title = Nokogiri::HTML(Net::HTTP.get(word)).title()
|
||||
title = title.gsub(/[\s]+/, " ").strip if not title.nil?
|
||||
|
||||
Scrape::Link.create(
|
||||
|
|
Loading…
Reference in New Issue