Merge branch 'master' of ssh://hackerspace.pl/ar/repost
Apparently i fucked up somethingmaster
commit
e43428af95
|
@ -2,3 +2,4 @@ config.rb
|
||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
*~
|
*~
|
||||||
|
Gemfile.lock
|
||||||
|
|
1
Gemfile
1
Gemfile
|
@ -6,3 +6,4 @@ gem "logger"
|
||||||
gem "nokogiri"
|
gem "nokogiri"
|
||||||
gem "datamapper"
|
gem "datamapper"
|
||||||
gem "dm-postgres-adapter"
|
gem "dm-postgres-adapter"
|
||||||
|
gem "dm-sqlite-adapter"
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
* Ratelimit outgoing messages
|
||||||
|
* Prettier config class, not just a Hash
|
||||||
|
* .split on [:params][1] in scrape, to make it work with multiple uris on one line
|
||||||
|
* somewhat done - prints title only for the first uri, to avoid flooding the channel
|
||||||
|
* Per channel/network configuration
|
||||||
|
* Multiple network support
|
||||||
|
* content downloading in scrape in a separate thread to not make the whole client timeout on downloading a big/slow url
|
||||||
|
* switch from open-uri and net/www to CURB (https://github.com/taf2/curb)
|
|
@ -2,5 +2,5 @@ require 'net/http'
|
||||||
require 'nokogiri'
|
require 'nokogiri'
|
||||||
|
|
||||||
Client.register_command("cycki") { |args|
|
Client.register_command("cycki") { |args|
|
||||||
Config[:commands][:cycki][:message] + Nokogiri::HTML(Net::HTTP.get(URI(Config[:commands][:cycki][:url]))).xpath('//img/@src').inner_text
|
Config[:commands][:cycki][:message] + Nokogiri::HTML(Net::HTTP.get(URI(Config[:commands][:cycki][:url]))).xpath('//img/@src').inner_text.gsub("_preview", "")
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,17 +1,60 @@
|
||||||
require 'open-uri'
|
require 'open-uri'
|
||||||
require 'digest/md5'
|
require 'digest/md5'
|
||||||
|
require 'data_mapper'
|
||||||
|
require 'nokogiri'
|
||||||
|
|
||||||
include EventMachine::IRC::Commands
|
include EventMachine::IRC::Commands
|
||||||
|
|
||||||
|
DataMapper.setup(:default, Config[:scrape][:db])
|
||||||
|
|
||||||
|
module Scrape
|
||||||
|
def self.download(uri)
|
||||||
|
ext = "." + uri.sub(/.*[.]([^.]*)/, '\1') if uri.match(/[^.]+[.][^.]+/)
|
||||||
|
content = open(uri).read
|
||||||
|
md5 = Digest::MD5.hexdigest(content)
|
||||||
|
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
||||||
|
file.write(content)
|
||||||
|
end
|
||||||
|
|
||||||
|
return md5 + ext
|
||||||
|
end
|
||||||
|
|
||||||
|
class Link
|
||||||
|
include DataMapper::Resource
|
||||||
|
|
||||||
|
property :id, Serial
|
||||||
|
property :url, String
|
||||||
|
property :original_url, String
|
||||||
|
property :created_at, DateTime
|
||||||
|
property :nick, String
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
DataMapper.auto_upgrade!
|
||||||
|
|
||||||
Client.register_trigger("PRIVMSG") do |msg|
|
Client.register_trigger("PRIVMSG") do |msg|
|
||||||
msg[:params].each do |word|
|
return 0 if msg[:params][1].nil?
|
||||||
if word =~ /4cdn[.]org/ then
|
msg[:params][1].split.each do |word|
|
||||||
ext = "." + word.sub(/.*[.]([^.]*)/, '\1') if word.match(/[^.]+[.][^.]+/)
|
return 0 if word =~ /notitle/
|
||||||
content = open(word).read
|
if word =~ /^https?:\/\// then
|
||||||
md5 = Digest::MD5.hexdigest(content)
|
original_url = word
|
||||||
File.open(Config[:scrape][:basepath] + md5 + ext, 'w') do |file|
|
if word =~ /4cdn[.]org/ then
|
||||||
file.write(content)
|
url = Scrape::download(word)
|
||||||
|
else
|
||||||
|
url = Digest::MD5.hexdigest(word)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
title = Nokogiri::HTML(open(word).read(10240)).title().split(/\n/)[0]
|
||||||
|
|
||||||
|
Scrape::Link.create(
|
||||||
|
:url => url,
|
||||||
|
:original_url => original_url,
|
||||||
|
:created_at => Time.now,
|
||||||
|
:nick => msg[:prefix].split('!').first
|
||||||
|
)
|
||||||
|
|
||||||
|
Client.privmsg(msg[:params][0], Config[:scrape][:title_prefix] + title) if not title.nil?
|
||||||
|
return 0
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue