now it actually scrapes some urls
parent
c71a306e09
commit
b813682d13
2
Gemfile
2
Gemfile
|
@ -4,3 +4,5 @@ gem "eventmachine"
|
||||||
gem "em-irc"
|
gem "em-irc"
|
||||||
gem "logger"
|
gem "logger"
|
||||||
gem "nokogiri"
|
gem "nokogiri"
|
||||||
|
gem "datamapper"
|
||||||
|
gem "dm-postgres-adapter"
|
||||||
|
|
|
@ -1,9 +1,17 @@
|
||||||
|
require 'open-uri'
|
||||||
|
require 'digest/md5'
|
||||||
|
|
||||||
include EventMachine::IRC::Commands
|
include EventMachine::IRC::Commands
|
||||||
|
|
||||||
Client.register_trigger("PRIVMSG") { |msg|
|
Client.register_trigger("PRIVMSG") { |msg|
|
||||||
message = msg[:params][1]
|
msg[:params].each { |word|
|
||||||
|
if word =~ /4cdn[.]org/ then
|
||||||
message.split { |word|
|
ext = "." + word.sub(/.*[.]([^.]*)/, '\1') if word.match(/[^.]+[.][^.]+/)
|
||||||
puts word if word ~ /4cdn[.]org/
|
content = open(word).read
|
||||||
|
md5 = Digest::MD5.hexdigest(content)
|
||||||
|
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') { |file|
|
||||||
|
file.write(content)
|
||||||
|
}
|
||||||
|
end
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue