now it actually scrapes some urls

This commit is contained in:
Robert Gerus 2014-03-09 19:40:32 +01:00
parent c71a306e09
commit b813682d13
2 changed files with 14 additions and 4 deletions

View file

@ -4,3 +4,5 @@ gem "eventmachine"
gem "em-irc"
gem "logger"
gem "nokogiri"
gem "datamapper"
gem "dm-postgres-adapter"

View file

@ -1,9 +1,17 @@
require 'open-uri'
require 'digest/md5'
include EventMachine::IRC::Commands
Client.register_trigger("PRIVMSG") { |msg|
message = msg[:params][1]
message.split { |word|
puts word if word ~ /4cdn[.]org/
msg[:params].each { |word|
if word =~ /4cdn[.]org/ then
ext = "." + word.sub(/.*[.]([^.]*)/, '\1') if word.match(/[^.]+[.][^.]+/)
content = open(word).read
md5 = Digest::MD5.hexdigest(content)
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') { |file|
file.write(content)
}
end
}
}