From 7d1c3470bb06e43245ccb7f067121de606506430 Mon Sep 17 00:00:00 2001 From: Leonardo Sandoval Date: Wed, 2 Sep 2015 08:59:43 +0000 Subject: distrodata: Take account proxies on distrodata tasks Proxies defined in the enviroment where not taken into account on the distrodata tasks. This commit implied passing the datastore into the distro_check library and context manager for the urllib.urlopen function. One way to run distrodata tasks is using 'universe' as target and the 'all' distrodata task: $ bitbake universe -c distrodataall $ bitbake universe -c distro_checkall $ bitbake universe -c checklicenseall Logs are located under TMPDIR/log [YOCTO #7567] Signed-off-by: Leonardo Sandoval Signed-off-by: Richard Purdie --- meta/classes/distrodata.bbclass | 6 +-- meta/lib/oe/distro_check.py | 113 +++++++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass index e6eb3f32b3..5a4c1b6faf 100644 --- a/meta/classes/distrodata.bbclass +++ b/meta/classes/distrodata.bbclass @@ -33,7 +33,7 @@ python do_distrodata_np() { tmpdir = d.getVar('TMPDIR', True) distro_check_dir = os.path.join(tmpdir, "distro_check") datetime = localdata.getVar('DATETIME', True) - dist_check.update_distro_data(distro_check_dir, datetime) + dist_check.update_distro_data(distro_check_dir, datetime, localdata) if pn.find("-native") != -1: pnstripped = pn.split("-native") @@ -118,7 +118,7 @@ python do_distrodata() { tmpdir = d.getVar('TMPDIR', True) distro_check_dir = os.path.join(tmpdir, "distro_check") datetime = localdata.getVar('DATETIME', True) - dist_check.update_distro_data(distro_check_dir, datetime) + dist_check.update_distro_data(distro_check_dir, datetime, localdata) pn = d.getVar("PN", True) bb.note("Package Name: %s" % pn) @@ -406,7 +406,7 @@ python do_distro_check() { bb.utils.mkdirhier(logpath) result_file = os.path.join(logpath, "distrocheck.csv") datetime = localdata.getVar('DATETIME', True) - dc.update_distro_data(distro_check_dir, datetime) + dc.update_distro_data(distro_check_dir, datetime, localdata) # do the comparison result = dc.compare_in_distro_packages_list(distro_check_dir, d) diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py index 8ed5b0ec80..b3419ce03a 100644 --- a/meta/lib/oe/distro_check.py +++ b/meta/lib/oe/distro_check.py @@ -1,7 +1,28 @@ -def get_links_from_url(url): +from contextlib import contextmanager +@contextmanager +def create_socket(url, d): + import urllib + socket = urllib.urlopen(url, proxies=get_proxies(d)) + try: + yield socket + finally: + socket.close() + +def get_proxies(d): + import os + proxykeys = ['HTTP_PROXY', 'http_proxy', + 'HTTPS_PROXY', 'https_proxy', + 'FTP_PROXY', 'ftp_proxy', + 'FTPS_PROXY', 'ftps_proxy', + 'NO_PROXY', 'no_proxy', + 'ALL_PROXY', 'all_proxy'] + proxyvalues = map(lambda key: d.getVar(key, True), proxykeys) + return dict(zip(proxykeys, proxyvalues)) + +def get_links_from_url(url, d): "Return all the href links found on the web location" - import urllib, sgmllib + import sgmllib class LinksParser(sgmllib.SGMLParser): def parse(self, s): @@ -24,19 +45,18 @@ def get_links_from_url(url): "Return the list of hyperlinks." return self.hyperlinks - sock = urllib.urlopen(url) - webpage = sock.read() - sock.close() + with create_socket(url,d) as sock: + webpage = sock.read() linksparser = LinksParser() linksparser.parse(webpage) return linksparser.get_hyperlinks() -def find_latest_numeric_release(url): +def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): try: release = float(link) except: @@ -70,7 +90,7 @@ def clean_package_list(package_list): return set.keys() -def get_latest_released_meego_source_package_list(): +def get_latest_released_meego_source_package_list(d): "Returns list of all the name os packages in the latest meego distro" package_names = [] @@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list(): package_list=clean_package_list(package_names) return "1.0", package_list -def get_source_package_list_from_url(url, section): +def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) - links = get_links_from_url(url) + links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) @@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section): return new_pkgs -def get_latest_released_fedora_source_package_list(): +def get_latest_released_fedora_source_package_list(d): "Returns list of all the name os packages in the latest fedora distro" - latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/") + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) - package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") + package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d) # package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") + package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def get_latest_released_opensuse_source_package_list(): +def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) + package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def get_latest_released_mandriva_source_package_list(): +def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") + latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) + package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) # package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") + package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def find_latest_debian_release(url): +def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" releases = [] - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): if link[:6] == "Debian": if ';' not in link: releases.append(link) @@ -143,16 +163,15 @@ def find_latest_debian_release(url): except: return "_NotFound_" -def get_debian_style_source_package_list(url, section): +def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - import urllib - sock = urllib.urlopen(url) - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - sock.close() - tmpfile.close() + with create_socket(url,d) as sock: + webpage = sock.read() + import tempfile + tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) + tmpfilename=tmpfile.name + tmpfile.write(sock.read()) + tmpfile.close() import gzip bb.note("Reading %s: %s" % (url, section)) @@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url, section): return package_names -def get_latest_released_debian_source_package_list(): +def get_latest_released_debian_source_package_list(d): "Returns list of all the name os packages in the latest debian distro" - latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" - package_names = get_debian_style_source_package_list(url, "main") + package_names = get_debian_style_source_package_list(url, "main", d) # url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" # package_names += get_debian_style_source_package_list(url, "contrib") url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates") + package_names += get_debian_style_source_package_list(url, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def find_latest_ubuntu_release(url): +def find_latest_ubuntu_release(url, d): "Find the latest listed ubuntu release on the given url" url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): if link[-8:] == "-updates": return link[:-8] return "_NotFound_" -def get_latest_released_ubuntu_source_package_list(): +def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest - package_names = get_debian_style_source_package_list(url, "main") + package_names = get_debian_style_source_package_list(url, "main", d) # url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest # package_names += get_debian_style_source_package_list(url, "multiverse") # url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest # package_names += get_debian_style_source_package_list(url, "universe") url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates") + package_names += get_debian_style_source_package_list(url, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def create_distro_packages_list(distro_check_dir): +def create_distro_packages_list(distro_check_dir, d): pkglst_dir = os.path.join(distro_check_dir, "package_lists") if not os.path.isdir (pkglst_dir): os.makedirs(pkglst_dir) @@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir): begin = datetime.now() for distro in per_distro_functions: name = distro[0] - release, package_list = distro[1]() + release, package_list = distro[1](d) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) package_list_file = os.path.join(pkglst_dir, name + "-" + release) f = open(package_list_file, "w+b") @@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir): delta = end - begin bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) -def update_distro_data(distro_check_dir, datetime): +def update_distro_data(distro_check_dir, datetime, d): """ If distro packages list data is old then rebuild it. The operations has to be protected by a lock so that @@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime): if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") - create_distro_packages_list(distro_check_dir) + create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) -- cgit 1.2.3-korg