285f991 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #182 from dalf/enh-proxy

[enh] image-proxy : handle ETag and date related headers, add hash to URL

commit285f991cd078db7df5622e3109984f1f3c651988 authorAdam Tauber <asciimoo@gmail.com> dateSun 18 Jan 2015, 10:39 (12 years ago) parenta865e66 1d51512

11 files changed+39−15

compare with master

unifiedsplit

searx/templates/courgette/result_templates/code.html+2−2 searx/templates/courgette/result_templates/videos.html+2−2 searx/templates/default/infobox.html+1−1 searx/templates/default/result_templates/code.html+1−1 searx/templates/default/result_templates/default.html+1−1 searx/templates/default/result_templates/map.html+1−1 searx/templates/default/result_templates/videos.html+1−1 searx/templates/oscar/infobox.html+1−1 searx/templates/oscar/result_templates/videos.html+1−1 searx/utils.py+8 searx/webapp.py+20−4

searx/templates/courgette/result_templates/code.html+2−2

@@ -1,9 +1,9 @@

11<div class="result {{ result.class }}">

22 <h3 class="result_title">{% if result['favicon'] %}<img width="14" height="14" class="favicon" src="static/{{theme}}/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}" />{% endif %}<a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

33 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}

4− <p class="content">{% if result.img_src %}<img src="{{ result.img_src }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

4+ <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

55 {% if result.repository %}<p class="content"><a href="{{ result.repository|safe }}">{{ result.repository }}</a></p>{% endif %}

66 {{ result.codelines|code_highlighter(result.code_language)|safe }}

88 <p class="url">{{ result.pretty_url }}</p>

9−</div>

9+</div>

searx/templates/courgette/result_templates/videos.html+2−2

@@ -5,6 +5,6 @@

66 <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

77 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span><br />{% endif %}

8− <a href="{{ result.url }}"><img width="400" src="{{ result.thumbnail }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}"/></a>

8+ <a href="{{ result.url }}"><img width="400" src="{{ image_proxify(result.thumbnail) }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}"/></a>

99 <p class="url">{{ result.pretty_url }}</p>

10−</div>

10+</div>

searx/templates/default/infobox.html+1−1

@@ -1,6 +1,6 @@

11<div class="infobox">

22 <h2>{{ infobox.infobox }}</h2>

3− {% if infobox.img_src %}<img src="{{ infobox.img_src }}" title="{{ infobox.infobox|striptags }}" alt="{{ infobox.infobox|striptags }}" />{% endif %}

3+ {% if infobox.img_src %}<img src="{{ image_proxify(infobox.img_src) }}" title="{{ infobox.infobox|striptags }}" alt="{{ infobox.infobox|striptags }}" />{% endif %}

44 <p>{{ infobox.entity }}</p>

55 <p>{{ infobox.content | safe }}</p>

66 {% if infobox.attributes %}

searx/templates/default/result_templates/code.html+1−1

@@ -2,7 +2,7 @@

22 <h3 class="result_title"> {% if result['favicon'] %}<img width="14" height="14" class="favicon" src="static/{{theme}}/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}" />{% endif %}<a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

33 <p class="url">{{ result.pretty_url }} <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}">cached</a></p>

44 {% if result.publishedDate %}<p class="published_date">{{ result.publishedDate }}</p>{% endif %}

5− <p class="content">{% if result.img_src %}<img src="{{ result.img_src }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

5+ <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

66 {% if result.repository %}<p class="result-content"><a href="{{ result.repository|safe }}">{{ result.repository }}</a></p>{% endif %}

88 {{ result.codelines|code_highlighter(result.code_language)|safe }}

searx/templates/default/result_templates/default.html+1−1

@@ -2,5 +2,5 @@

22 <h3 class="result_title">{% if "icon_"~result.engine~".ico" in favicons %}<img width="14" height="14" class="favicon" src="{{ url_for('static', filename='img/icons/icon_'+result.engine+'.ico') }}" alt="{{result.engine}}" />{% endif %}<a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

33 <p class="url">{{ result.pretty_url }} <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}">cached</a>

44 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}</p>

5− <p class="content">{% if result.img_src %}<img src="{{ result.img_src }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

66</div>

searx/templates/default/result_templates/map.html+1−1

@@ -8,6 +8,6 @@

88 <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

99 <p class="url">{{ result.pretty_url }} <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}">cached</a>

1010 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}</p>

11− <p class="content">{% if result.img_src %}<img src="{{ result.img_src }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

11+ <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

1212 </div>

1313</div>

searx/templates/default/result_templates/videos.html+1−1

@@ -1,6 +1,6 @@

11<div class="result">

33 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span><br />{% endif %}

4− <a href="{{ result.url }}"><img class="thumbnail" src="{{ result.thumbnail }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}"/></a>

4+ <a href="{{ result.url }}"><img class="thumbnail" src="{{ image_proxify(result.thumbnail) }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}"/></a>

55 <p class="url">{{ result.url }}</p>

66</div>

searx/templates/oscar/infobox.html+1−1

@@ -3,7 +3,7 @@

33 <h4 class="panel-title">{{ infobox.infobox }}</h4>

44 </div>

55 <div class="panel-body">

6− {% if infobox.img_src %}<img class="img-responsive center-block infobox_part" src="{{ infobox.img_src }}" alt="{{ infobox.infobox }}" />{% endif %}

6+ {% if infobox.img_src %}<img class="img-responsive center-block infobox_part" src="{{ image_proxify(infobox.img_src) }}" alt="{{ infobox.infobox }}" />{% endif %}

77 {% if infobox.content %}<p class="infobox_part">{{ infobox.content }}</p>{% endif %}

99 {% if infobox.attributes %}

searx/templates/oscar/result_templates/videos.html+1−1

@@ -15,7 +15,7 @@

1515

1616<div class="container-fluid">

1717 <div class="row">

18− <a href="{{ result.url }}"><img class="thumbnail col-xs-6 col-sm-4 col-md-4 result-content" src="{{ result.thumbnail|safe }}" alt="{{ result.title|striptags }} {{ result.engine }}" /></a>

18+ <a href="{{ result.url }}"><img class="thumbnail col-xs-6 col-sm-4 col-md-4 result-content" src="{{ image_proxify(result.thumbnail) }}" alt="{{ result.title|striptags }} {{ result.engine }}" /></a>

1919 {% if result.content %}<p class="col-xs-12 col-sm-8 col-md-8 result-content">{{ result.content|safe }}</p>{% endif %}

2020 </div>

2121</div>

searx/utils.py+8

@@ -206,3 +206,11 @@ def format_date_by_locale(date_string, locale_string):

206206 except:

207207 logger.warning('cannot set original locale: {0}'.format(orig_locale))

208208 return formatted_date

209+

210+

211+def dict_subset(d, properties):

212+ result = {}

213+ for k in properties:

214+ if k in d:

215+ result[k] = d[k]

216+ return result

searx/webapp.py+20−4

@@ -25,6 +25,7 @@ if __name__ == '__main__':

2525import json

2626import cStringIO

2727import os

28+import hashlib

2829

2930from datetime import datetime, timedelta

3031from requests import get as http_get

@@ -41,7 +42,7 @@ from searx.engines import (

4142)

4243from searx.utils import (

4344 UnicodeWriter, highlight_content, html_to_text, get_themes,

44− get_static_files, get_result_templates, gen_useragent

45+ get_static_files, get_result_templates, gen_useragent, dict_subset

4546)

4647from searx.version import VERSION_STRING

4748from searx.languages import language_codes

@@ -216,8 +217,10 @@ def image_proxify(url):

216217 if not settings['server'].get('image_proxy') and not request.cookies.get('image_proxy'):

217218 return url

218219

220+ h = hashlib.sha256(url + settings['server']['secret_key']).hexdigest()

221+

219222 return '{0}?{1}'.format(url_for('image_proxy'),

220− urlencode(dict(url=url)))

223+ urlencode(dict(url=url, h=h)))

221224

222225

223226def render(template_name, override_theme=None, **kwargs):

@@ -562,10 +565,21 @@ def image_proxy():

562565 if not url:

563566 return '', 400

564567

568+ h = hashlib.sha256(url + settings['server']['secret_key']).hexdigest()

569+

570+ if h != request.args.get('h'):

571+ return '', 400

572+

573+ headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

574+ headers['User-Agent'] = gen_useragent()

575+

565576 resp = http_get(url,

566577 stream=True,

567578 timeout=settings['server'].get('request_timeout', 2),

568− headers={'User-Agent': gen_useragent()})

579+ headers=headers)

580+

581+ if resp.status_code == 304:

582+ return '', resp.status_code

569583

570584 if resp.status_code != 200:

571585 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

@@ -586,7 +600,9 @@ def image_proxy():

586600 return '', 502 # Bad gateway - file is too big (>5M)

587601 img += chunk

588602

589− return Response(img, mimetype=resp.headers['content-type'])

603+ headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

604+

605+ return Response(img, mimetype=resp.headers['content-type'], headers=headers)

590606

591607

592608@app.route('/stats', methods=['GET'])