@@ -1,30 +1,63 @@
1+## Twitter (Social media)
2+#
3+# @website https://www.bing.com/news
4+# @provide-api yes (https://dev.twitter.com/docs/using-search)
5+#
6+# @using-api no
7+# @results HTML (using search portal)
8+# @stable no (HTML can change)
9+# @parse url, title, content
10+#
11+# @todo publishedDate
12+
113from urlparse import urljoin
214from urllib import urlencode
315from lxml import html
416from cgi import escape
517
18+# engine dependent config
619categories = ['social media']
20+language_support = True
721
22+# search-url
823base_url = 'https://twitter.com/'
924search_url = base_url+'search?'
25+
26+# specific xpath variables
27+results_xpath = '//li[@data-item-type="tweet"]'
28+link_xpath = './/small[@class="time"]//a'
1029title_xpath = './/span[@class="username js-action-profile-name"]//text()'
1130content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'
1231
1332
33+# do search-request
1434def request(query, params):
1535 params['url'] = search_url + urlencode({'q': query})
36+
37+ # set language if specified
38+ if params['language'] != 'all':
39+ params['cookies']['lang'] = params['language'].split('_')[0]
40+
1641 return params
1742
1843
44+# get response from search-request
1945def response(resp):
2046 results = []
47+
2148 dom = html.fromstring(resp.text)
22− for tweet in dom.xpath('//li[@data-item-type="tweet"]'):
23− link = tweet.xpath('.//small[@class="time"]//a')[0]
49+
50+ # parse results
51+ for tweet in dom.xpath(results_xpath):
52+ link = tweet.xpath(link_xpath)[0]
2453 url = urljoin(base_url, link.attrib.get('href'))
2554 title = ''.join(tweet.xpath(title_xpath))
2655 content = escape(''.join(tweet.xpath(content_xpath)))
56+
57+ # append result
2758 results.append({'url': url,
2859 'title': title,
2960 'content': content})
61+
62+ # return results
3063 return results