165 lines
5.7 KiB
Python
165 lines
5.7 KiB
Python
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# #
|
|
# Feeds updater for iSida Jabber Bot #
|
|
# Copyright (C) 2011 diSabler <dsy@dsy.name> #
|
|
# #
|
|
# This program is free software: you can redistribute it and/or modify #
|
|
# it under the terms of the GNU General Public License as published by #
|
|
# the Free Software Foundation, either version 3 of the License, or #
|
|
# (at your option) any later version. #
|
|
# #
|
|
# This program is distributed in the hope that it will be useful, #
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
|
|
# GNU General Public License for more details. #
|
|
# #
|
|
# You should have received a copy of the GNU General Public License #
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>. #
|
|
# #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
import os,urllib2,sys,re,chardet,hashlib
|
|
|
|
def smart_encode(text,enc):
|
|
tx,splitter = '','|'
|
|
while text.count(splitter): splitter += '|'
|
|
ttext = text.replace('</','<%s/' % splitter).split(splitter)
|
|
for tmp in ttext:
|
|
try: tx += unicode(tmp,enc)
|
|
except: pass
|
|
return tx
|
|
|
|
def html_encode(body):
|
|
encidx = body.find('encoding=')
|
|
if encidx >= 0:
|
|
enc = body[encidx+10:encidx+30]
|
|
if enc.count('"'): enc = enc[:enc.find('"')]
|
|
elif enc.count('\''): enc = enc[:enc.find('\'')]
|
|
elif enc.count('&'): enc = enc[:enc.find('&')]
|
|
else:
|
|
encidx = body.find('charset=')
|
|
if encidx >= 0:
|
|
enc = body[encidx+8:encidx+30]
|
|
if enc.count('"'): enc = enc[:enc.find('"')]
|
|
elif enc.count('\''): enc = enc[:enc.find('\'')]
|
|
elif enc.count('&'): enc = enc[:enc.find('&')]
|
|
else: enc = chardet.detect(body)['encoding']
|
|
if body == None: body = ''
|
|
if enc == None or enc == '' or enc.lower() == 'unicode': enc = 'utf-8'
|
|
if enc == 'ISO-8859-2':
|
|
tx,splitter = '','|'
|
|
while body.count(splitter): splitter += '|'
|
|
tbody = body.replace('</','<'+splitter+'/').split(splitter)
|
|
cntr = 0
|
|
for tmp in tbody:
|
|
try:
|
|
enc = chardet.detect(tmp)['encoding']
|
|
if enc == None or enc == '' or enc.lower() == 'unicode': enc = 'utf-8'
|
|
tx += unicode(tmp,enc)
|
|
except:
|
|
ttext = ''
|
|
for tmp2 in tmp:
|
|
if (tmp2<='~'): ttext+=tmp2
|
|
else: ttext+='?'
|
|
tx += ttext
|
|
cntr += 1
|
|
return tx
|
|
else:
|
|
try: return smart_encode(body,enc)
|
|
except: return 'Encoding error!'
|
|
|
|
def readfile(filename):
|
|
fp = file(filename)
|
|
data = fp.read()
|
|
fp.close()
|
|
return data
|
|
|
|
def writefile(filename, data):
|
|
fp = file(filename, 'w')
|
|
fp.write(data)
|
|
fp.close()
|
|
|
|
def getFile(filename,default):
|
|
if os.path.isfile(filename):
|
|
try: filebody = eval(readfile(filename))
|
|
except:
|
|
if os.path.isfile(filename+'.back'):
|
|
while True:
|
|
try:
|
|
filebody = eval(readfile(filename+'.back'))
|
|
break
|
|
except: pass
|
|
else:
|
|
filebody = default
|
|
writefile(filename,str(default))
|
|
else:
|
|
filebody = default
|
|
writefile(filename,str(default))
|
|
writefile(filename+'.back',str(filebody))
|
|
return filebody
|
|
|
|
def get_tag(body,tag):
|
|
T = re.findall('<%s.*?>(.*?)</%s>' % (tag,tag),body,re.S)
|
|
if T: return T[0]
|
|
else: return ''
|
|
|
|
# --------- Settings ----------
|
|
set_folder = 'settings/'
|
|
feeds = set_folder+'feed'
|
|
user_agent = 'Mozilla/5.0 (X11; U; Linux x86_64; ru; rv:1.9.0.4) Gecko/2008120916 Gentoo Firefox/3.0.4'
|
|
# -----------------------------
|
|
|
|
feedbase = getFile(feeds,[])
|
|
newfeed = []
|
|
for fdd in feedbase:
|
|
link = fdd[0]
|
|
print link,
|
|
if not link[:10].count('://'): link = 'http://'+link
|
|
try:
|
|
if float(sys.version[:3]) >= 2.6: # python 2.6 and higher
|
|
req = urllib2.Request(link.encode('utf-8'))
|
|
req.add_header('User-Agent','user_agent')
|
|
feed = urllib2.urlopen(url=req,timeout=30).read(262144)
|
|
else: feed = urllib.urlopen(link).read()
|
|
except: feed = 'Unable to access server!'
|
|
is_rss_aton,fc = 0,feed[:256]
|
|
if fc.count('<?xml version='):
|
|
if fc.count('<feed'):
|
|
is_rss_aton = 2
|
|
t_feed = feed.split('<title>')
|
|
feed = t_feed[0]
|
|
for tmp in t_feed[1:]:
|
|
tm = tmp.split('</title>',1)
|
|
if ord(tm[0][-1]) == 208: tm[0] = tm[0][:-1] + '...'
|
|
feed += '<title>%s</title>%s' % tuple(tm)
|
|
elif fc.count('<rss') or fc.count('<rdf'): is_rss_aton = 1
|
|
feed = html_encode(feed)
|
|
feed = re.sub(u'(<span.*?>.*?</span>)','',feed)
|
|
feed = re.sub(u'(<div.*?>)','',feed)
|
|
feed = re.sub(u'(</div>)','',feed)
|
|
if is_rss_aton and feed != 'Encoding error!' and feed != 'Unable to access server!':
|
|
print '... readed',
|
|
if is_rss_aton == 1:
|
|
if feed.count('<item>'): fd = feed.split('<item>')
|
|
else: fd = feed.split('<item ')
|
|
feed = [fd[0]]
|
|
for tmp in fd[1:]: feed.append(tmp.split('</item>')[0])
|
|
else:
|
|
if feed.count('<entry>'): fd = feed.split('<entry>')
|
|
else: fd = feed.split('<entry ')
|
|
feed = [fd[0]]
|
|
for tmp in fd[1:]: feed.append(tmp.split('</entry>')[0])
|
|
break_point = []
|
|
for tmp in feed[1:]:
|
|
ttitle = get_tag(tmp,'title').replace('<br>','\n')
|
|
break_point.append(hashlib.md5(ttitle.encode('utf-8')).hexdigest())
|
|
newfeed.append([link,fdd[1],fdd[2],fdd[3],fdd[4],break_point])
|
|
print '... generated'
|
|
else: print '... fault!'
|
|
writefile(feeds,str(newfeed))
|
|
print 'done!'
|
|
|
|
# The end is near! |