fix: exception in rss
This commit is contained in:
diSabler
@@ -90,6 +90,12 @@ def clients_stats(type, jid, nick, text):
|
|||||||
elif text: match = '%%%s%%' % reduce_spaces_all(' '.join(text))
|
elif text: match = '%%%s%%' % reduce_spaces_all(' '.join(text))
|
||||||
else: is_short = is_global = is_os = is_user = False
|
else: is_short = is_global = is_os = is_user = False
|
||||||
|
|
||||||
|
# global os short
|
||||||
|
# select split_part(replace(replace(replace(replace(replace(replace(replace(lower(os),'/',' '),'"',''),'microsoft',''),'(c)',''),'©',''),'name=',''),'none',''),' ',1) as oss, count(*) as coss from versions group by oss order by coss desc;
|
||||||
|
|
||||||
|
# global short
|
||||||
|
# select split_part(client,' ',1) as clients, count(*) as cclients from versions group by clients order by cclients desc;
|
||||||
|
|
||||||
if is_global:
|
if is_global:
|
||||||
if is_user: req,par = 'select client,version,os from versions where jid=%s and (client ilike %s or version ilike %s or os ilike %s)',(cjid,match,match,match)
|
if is_user: req,par = 'select client,version,os from versions where jid=%s and (client ilike %s or version ilike %s or os ilike %s)',(cjid,match,match,match)
|
||||||
else: req,par = 'select client,version,os from versions where client ilike %s or version ilike %s or os ilike %s',(match,match,match)
|
else: req,par = 'select client,version,os from versions where client ilike %s or version ilike %s or os ilike %s',(match,match,match)
|
||||||
|
|||||||
236
plugins/main.py
236
plugins/main.py
@@ -1308,127 +1308,129 @@ def rss(type, jid, nick, text):
|
|||||||
elif mode in ['new','get']:
|
elif mode in ['new','get']:
|
||||||
link = text[1]
|
link = text[1]
|
||||||
if not re.findall('^http(s?)://',link[:10]): link = 'http://%s' % link
|
if not re.findall('^http(s?)://',link[:10]): link = 'http://%s' % link
|
||||||
body, result = get_opener(enidna(link))
|
if cur_execute_fetchone('select * from feed where room=%s and url=%s;',(jid,link)):
|
||||||
modified,need_update_feed = '',True
|
body, result = get_opener(enidna(link))
|
||||||
if result:
|
modified,need_update_feed = '',True
|
||||||
modified = body.headers.get('last-modified',None)
|
if result:
|
||||||
if not modified: modified = get_tag(load_page_size(link, 1024),'updated')
|
modified = body.headers.get('last-modified',None)
|
||||||
if modified:
|
if not modified: modified = get_tag(load_page_size(link, 1024),'updated')
|
||||||
changed = cur_execute_fetchone('select changed from feed where room=%s and url=%s;',(jid,link))[0]
|
if modified:
|
||||||
if modified == changed: need_update_feed = False
|
changed = cur_execute_fetchone('select changed from feed where room=%s and url=%s;',(jid,link))[0]
|
||||||
else: cur_execute('update feed set changed=%s where room=%s and url=%s',(modified,jid,link))
|
if modified == changed: need_update_feed = False
|
||||||
is_rss_aton = 0
|
else: cur_execute('update feed set changed=%s where room=%s and url=%s',(modified,jid,link))
|
||||||
if need_update_feed:
|
is_rss_aton = 0
|
||||||
try:
|
if need_update_feed:
|
||||||
req = urllib2.Request(link.encode('utf-8'))
|
try:
|
||||||
req.add_header('User-Agent',GT('user_agent'))
|
req = urllib2.Request(link.encode('utf-8'))
|
||||||
feed = urllib2.urlopen(url=req,timeout=GT('rss_get_timeout')).read(GT('size_overflow'))
|
req.add_header('User-Agent',GT('user_agent'))
|
||||||
except: feed = L('Unable to access server!','%s/%s'%(jid,nick))
|
feed = urllib2.urlopen(url=req,timeout=GT('rss_get_timeout')).read(GT('size_overflow'))
|
||||||
fc = feed[:256]
|
except: feed = L('Unable to access server!','%s/%s'%(jid,nick))
|
||||||
if '<?xml version=' in fc:
|
fc = feed[:256]
|
||||||
if '<feed' in fc:
|
if '<?xml version=' in fc:
|
||||||
is_rss_aton = 2
|
if '<feed' in fc:
|
||||||
t_feed = feed.split('<title>')
|
is_rss_aton = 2
|
||||||
feed = t_feed[0]
|
t_feed = feed.split('<title>')
|
||||||
for tmp in t_feed[1:]:
|
feed = t_feed[0]
|
||||||
tm = tmp.split('</title>',1)
|
for tmp in t_feed[1:]:
|
||||||
if ord(tm[0][-1]) == 208: tm[0] = tm[0][:-1] + '...'
|
tm = tmp.split('</title>',1)
|
||||||
feed += '<title>%s</title>%s' % tuple(tm)
|
if ord(tm[0][-1]) == 208: tm[0] = tm[0][:-1] + '...'
|
||||||
elif '<rss' in fc or '<rdf' in fc: is_rss_aton = 1
|
feed += '<title>%s</title>%s' % tuple(tm)
|
||||||
feed = html_encode(feed)
|
elif '<rss' in fc or '<rdf' in fc: is_rss_aton = 1
|
||||||
feed = re.sub('(<span.*?>.*?</span>)','',feed)
|
feed = html_encode(feed)
|
||||||
feed = re.sub('(<div.*?>)','',feed)
|
feed = re.sub('(<span.*?>.*?</span>)','',feed)
|
||||||
feed = re.sub('(</div>)','',feed)
|
feed = re.sub('(<div.*?>)','',feed)
|
||||||
else: feed = L('New feeds not found!','%s/%s'%(jid,nick))
|
feed = re.sub('(</div>)','',feed)
|
||||||
if is_rss_aton and feed != L('Encoding error!','%s/%s'%(jid,nick)) and feed != L('Unable to access server!','%s/%s'%(jid,nick)):
|
else: feed = L('New feeds not found!','%s/%s'%(jid,nick))
|
||||||
if is_rss_aton == 1:
|
if is_rss_aton and feed != L('Encoding error!','%s/%s'%(jid,nick)) and feed != L('Unable to access server!','%s/%s'%(jid,nick)):
|
||||||
if '<item>' in feed: fd = feed.split('<item>')
|
if is_rss_aton == 1:
|
||||||
else: fd = feed.split('<item ')
|
if '<item>' in feed: fd = feed.split('<item>')
|
||||||
feed = [fd[0]]
|
else: fd = feed.split('<item ')
|
||||||
for tmp in fd[1:]: feed.append(tmp.split('</item>')[0])
|
feed = [fd[0]]
|
||||||
else:
|
for tmp in fd[1:]: feed.append(tmp.split('</item>')[0])
|
||||||
if '<entry>' in feed: fd = feed.split('<entry>')
|
else:
|
||||||
else: fd = feed.split('<entry ')
|
if '<entry>' in feed: fd = feed.split('<entry>')
|
||||||
feed = [fd[0]]
|
else: fd = feed.split('<entry ')
|
||||||
for tmp in fd[1:]: feed.append(tmp.split('</entry>')[0])
|
feed = [fd[0]]
|
||||||
if len(text) > 2 and text[2].isdigit(): lng = int(text[2])
|
for tmp in fd[1:]: feed.append(tmp.split('</entry>')[0])
|
||||||
else: lng = len(feed)-1
|
if len(text) > 2 and text[2].isdigit(): lng = int(text[2])
|
||||||
if len(feed)-1 <= lng: lng = len(feed)-1
|
else: lng = len(feed)-1
|
||||||
if lng > GT('rss_max_feed_limit'): lng = GT('rss_max_feed_limit')
|
if len(feed)-1 <= lng: lng = len(feed)-1
|
||||||
elif len < 1: lng = 1
|
if lng > GT('rss_max_feed_limit'): lng = GT('rss_max_feed_limit')
|
||||||
if len(text) > 3: submode = text[3]
|
elif len < 1: lng = 1
|
||||||
else: submode = 'full'
|
if len(text) > 3: submode = text[3]
|
||||||
msg = L('Feeds for','%s/%s'%(jid,nick))+' '
|
else: submode = 'full'
|
||||||
if 'url' in submode.split('-'): submode,urlmode = submode.split('-')[0],True
|
msg = L('Feeds for','%s/%s'%(jid,nick))+' '
|
||||||
else:
|
if 'url' in submode.split('-'): submode,urlmode = submode.split('-')[0],True
|
||||||
urlmode = None
|
else:
|
||||||
msg += link+' '
|
urlmode = None
|
||||||
msg += get_tag(feed[0],'title')
|
msg += link+' '
|
||||||
try:
|
msg += get_tag(feed[0],'title')
|
||||||
break_point = []
|
try:
|
||||||
for tmp in feed[1:GT('rss_max_feed_limit')+1]:
|
break_point = []
|
||||||
ttitle = get_tag(tmp,'title').replace('<br>','\n')
|
for tmp in feed[1:GT('rss_max_feed_limit')+1]:
|
||||||
break_point.append(hashlib.md5(ttitle.encode('utf-8')).hexdigest())
|
ttitle = get_tag(tmp,'title').replace('<br>','\n')
|
||||||
tstop = rss_flush(jid,link,break_point)
|
break_point.append(hashlib.md5(ttitle.encode('utf-8')).hexdigest())
|
||||||
t_msg, new_count = [], 0
|
tstop = rss_flush(jid,link,break_point)
|
||||||
for mmsg in feed[1:GT('rss_max_feed_limit')+1]:
|
t_msg, new_count = [], 0
|
||||||
ttitle = get_tag(mmsg,'title').replace('<br>','\n')
|
for mmsg in feed[1:GT('rss_max_feed_limit')+1]:
|
||||||
if mode == 'get' or not (hashlib.md5(ttitle.encode('utf-8')).hexdigest() in tstop):
|
ttitle = get_tag(mmsg,'title').replace('<br>','\n')
|
||||||
if is_rss_aton == 1: tbody,turl = get_tag(mmsg,'description').replace('<br>','\n'),get_tag(mmsg,'link')
|
if mode == 'get' or not (hashlib.md5(ttitle.encode('utf-8')).hexdigest() in tstop):
|
||||||
else:
|
if is_rss_aton == 1: tbody,turl = get_tag(mmsg,'description').replace('<br>','\n'),get_tag(mmsg,'link')
|
||||||
tbody = get_tag(mmsg,'content').replace('<br>','\n')
|
else:
|
||||||
try:
|
tbody = get_tag(mmsg,'content').replace('<br>','\n')
|
||||||
tu1 = mmsg.find('href=\"',mmsg.index('<link'))+6
|
try:
|
||||||
tu2 = mmsg.find('\"',tu1)
|
tu1 = mmsg.find('href=\"',mmsg.index('<link'))+6
|
||||||
turl = mmsg[tu1:tu2].replace('<br>','\n')
|
tu2 = mmsg.find('\"',tu1)
|
||||||
except: turl = 'URL %s' % L('Not found!','%s/%s'%(jid,nick))
|
turl = mmsg[tu1:tu2].replace('<br>','\n')
|
||||||
tsubj,tmsg,tlink = '','',''
|
except: turl = 'URL %s' % L('Not found!','%s/%s'%(jid,nick))
|
||||||
if submode == 'full': tsubj,tmsg = replacer(ttitle),replacer(tbody.replace('<li>',u'\n♦ ').replace('</li>',''))
|
tsubj,tmsg,tlink = '','',''
|
||||||
elif submode == 'body': tmsg = replacer(tbody.replace('<li>',u'\n♦ ').replace('</li>',''))
|
if submode == 'full': tsubj,tmsg = replacer(ttitle),replacer(tbody.replace('<li>',u'\n♦ ').replace('</li>',''))
|
||||||
elif submode == 'head': tsubj = replacer(ttitle)
|
elif submode == 'body': tmsg = replacer(tbody.replace('<li>',u'\n♦ ').replace('</li>',''))
|
||||||
else: return
|
elif submode == 'head': tsubj = replacer(ttitle)
|
||||||
if urlmode: tlink = urllib.unquote(turl.encode('utf8')).decode('utf8','ignore')
|
else: return
|
||||||
t_msg.append((tsubj.replace('\n','; '),smart_concat(tmsg),tlink))
|
if urlmode: tlink = urllib.unquote(turl.encode('utf8')).decode('utf8','ignore')
|
||||||
new_count += 1
|
t_msg.append((tsubj.replace('\n','; '),smart_concat(tmsg),tlink))
|
||||||
if new_count >= lng: break
|
new_count += 1
|
||||||
if new_count:
|
if new_count >= lng: break
|
||||||
t_msg.reverse()
|
if new_count:
|
||||||
tmp = ''
|
t_msg.reverse()
|
||||||
for tm in t_msg: tmp += '!'.join(tm)
|
tmp = ''
|
||||||
if len(tmp+msg)+len(t_msg)*12 >= msg_limit:
|
for tm in t_msg: tmp += '!'.join(tm)
|
||||||
over = 100 * msg_limit / (len(tmp+msg)+len(t_msg)*12.0) # overflow in persent
|
if len(tmp+msg)+len(t_msg)*12 >= msg_limit:
|
||||||
tt_msg = []
|
over = 100 * msg_limit / (len(tmp+msg)+len(t_msg)*12.0) # overflow in persent
|
||||||
|
tt_msg = []
|
||||||
|
for tm in t_msg:
|
||||||
|
tsubj,tmsg,tlink = tm
|
||||||
|
cut = int(len(tsubj+tmsg+tlink)/100*over)
|
||||||
|
if cut < len(tlink): tsubj,tmsg,tlink = u'%s[…]' % tsubj[:cut],'',''
|
||||||
|
elif cut < len(tsubj+tlink): tsubj,tmsg = u'%s[…]' % tsubj[:cut-len(tlink)],''
|
||||||
|
else: tmsg = u'%s[…]' % tmsg[:cut-len(tlink+tsubj)]
|
||||||
|
tt_msg.append((tsubj,tmsg,tlink))
|
||||||
|
t_msg = tt_msg
|
||||||
|
tmp = ''
|
||||||
for tm in t_msg:
|
for tm in t_msg:
|
||||||
tsubj,tmsg,tlink = tm
|
if submode == 'full': tmp += u'\n• %s\n%s' % tm[0:2]
|
||||||
cut = int(len(tsubj+tmsg+tlink)/100*over)
|
elif submode == 'body': tmp += u'\n• %s' % tm[1]
|
||||||
if cut < len(tlink): tsubj,tmsg,tlink = u'%s[…]' % tsubj[:cut],'',''
|
elif submode == 'head': tmp += u'\n• %s' % tm[0]
|
||||||
elif cut < len(tsubj+tlink): tsubj,tmsg = u'%s[…]' % tsubj[:cut-len(tlink)],''
|
if len(tm[2]): tmp += '\n'+tm[2]
|
||||||
else: tmsg = u'%s[…]' % tmsg[:cut-len(tlink+tsubj)]
|
msg = unhtml(msg+tmp)
|
||||||
tt_msg.append((tsubj,tmsg,tlink))
|
elif mode == 'new':
|
||||||
t_msg = tt_msg
|
if text[4] == 'silent': nosend = True
|
||||||
tmp = ''
|
else: msg = L('New feeds not found!','%s/%s'%(jid,nick))
|
||||||
for tm in t_msg:
|
except Exception,SM:
|
||||||
if submode == 'full': tmp += u'\n• %s\n%s' % tm[0:2]
|
rss_flush(jid,link,None)
|
||||||
elif submode == 'body': tmp += u'\n• %s' % tm[1]
|
|
||||||
elif submode == 'head': tmp += u'\n• %s' % tm[0]
|
|
||||||
if len(tm[2]): tmp += '\n'+tm[2]
|
|
||||||
msg = unhtml(msg+tmp)
|
|
||||||
elif mode == 'new':
|
|
||||||
if text[4] == 'silent': nosend = True
|
if text[4] == 'silent': nosend = True
|
||||||
else: msg = L('New feeds not found!','%s/%s'%(jid,nick))
|
else: msg = L('Error! %s','%s/%s'%(jid,nick)) % SM
|
||||||
except Exception,SM:
|
else:
|
||||||
rss_flush(jid,link,None)
|
rss_flush(jid,link,None)
|
||||||
if text[4] == 'silent': nosend = True
|
if text[4] == 'silent': nosend = True
|
||||||
else: msg = L('Error! %s','%s/%s'%(jid,nick)) % SM
|
else:
|
||||||
else:
|
if need_update_feed:
|
||||||
rss_flush(jid,link,None)
|
if feed in [L('Encoding error!','%s/%s'%(jid,nick)),L('Unable to access server!','%s/%s'%(jid,nick))]: title = feed
|
||||||
if text[4] == 'silent': nosend = True
|
else: title = html_encode(get_tag(feed,'title'))
|
||||||
else:
|
msg = L('Bad url or rss/atom not found at %s - %s','%s/%s'%(jid,nick)) % (link,title)
|
||||||
if need_update_feed:
|
else: msg = feed
|
||||||
if feed in [L('Encoding error!','%s/%s'%(jid,nick)),L('Unable to access server!','%s/%s'%(jid,nick))]: title = feed
|
else: msg = L('Not found!','%s/%s'%(jid,nick))
|
||||||
else: title = html_encode(get_tag(feed,'title'))
|
|
||||||
msg = L('Bad url or rss/atom not found at %s - %s','%s/%s'%(jid,nick)) % (link,title)
|
|
||||||
else: msg = feed
|
|
||||||
if not nosend: send_msg(type, jid, nick, msg)
|
if not nosend: send_msg(type, jid, nick, msg)
|
||||||
|
|
||||||
def configure(type, jid, nick, text):
|
def configure(type, jid, nick, text):
|
||||||
|
|||||||
Reference in New Issue
Block a user