git.iko.soy/searx
on update_data_update_firefox_version.py help sign in
searx/searx_extra/update/update_firefox_version.py
68 lines 2.0 kB history raw
1#!/usr/bin/env python
3import json
4import requests
5import re
6from os.path import dirname, join
7from urllib.parse import urlparse, urljoin
8from distutils.version import LooseVersion, StrictVersion
9from lxml import html
10from searx import searx_dir
12URL = 'https://ftp.mozilla.org/pub/firefox/releases/'
13RELEASE_PATH = '/pub/firefox/releases/'
15NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?$')
16# BETA_REGEX = re.compile('.*[0-9]b([0-9\-a-z]+)$')
17# ESR_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?esr$')
19#
20useragents = {
21 "versions": (),
22 "os": ('Windows NT 10.0; WOW64',
23 'X11; Linux x86_64'),
24 "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
28def fetch_firefox_versions():
29 resp = requests.get(URL, timeout=2.0)
30 if resp.status_code != 200:
31 raise Exception("Error fetching firefox versions, HTTP code " + resp.status_code)
32 else:
33 dom = html.fromstring(resp.text)
34 versions = []
36 for link in dom.xpath('//a/@href'):
37 url = urlparse(urljoin(URL, link))
38 path = url.path
39 if path.startswith(RELEASE_PATH):
40 version = path[len(RELEASE_PATH):-1]
41 if NORMAL_REGEX.match(version):
42 versions.append(LooseVersion(version))
44 list.sort(versions, reverse=True)
45 return versions
48def fetch_firefox_last_versions():
49 versions = fetch_firefox_versions()
51 result = []
52 major_last = versions[0].version[0]
53 major_list = (major_last, major_last - 1)
54 for version in versions:
55 major_current = version.version[0]
56 if major_current in major_list:
57 result.append(version.vstring)
59 return result
62def get_useragents_filename():
63 return join(join(searx_dir, "data"), "useragents.json")
66useragents["versions"] = fetch_firefox_last_versions()
67with open(get_useragents_filename(), "w") as f:
68 json.dump(useragents, f, indent=4, ensure_ascii=False)