def create_socket(url, d): import urllib from bb.utils import export_proxies export_proxies(d) return urllib.request.urlopen(url) def get_links_from_url(url, d): "Return all the href links found on the web location" from bs4 import BeautifulSoup, SoupStrainer soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) hyperlinks = [] for line in soup.find_all('a', href=True): hyperlinks.append(line['href'].strip('/')) return hyperlinks def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" for link in get_links_from_url(url, d): try: # TODO use LooseVersion release = float(link) except: release = 0 if release > max: max = release maxstr = link return maxstr def is_src_rpm(name): "Check if the link is pointing to a src.rpm file" return name.endswith(".src.rpm") def package_name_from_srpm(srpm): "Strip out the package name from the src.rpm filename" # ca-certificates-2016.2.7-1.0.fc24.src.rpm # ^name ^ver ^release^removed (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) return name def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) new_pkgs = set() for pkgs in names_list: new_pkgs.add(pkgs + ":" + section) return new_pkgs def get_source_package_list_from_url_by_letter(url, section, d): import string from urllib.error import HTTPError packages = set() for letter in (string.ascii_lowercase + string.digits): # Not all subfolders may exist, so silently handle 404 try: packages |= get_source_package_list_from_url(url + "/" + letter, section, d) except HTTPError as e: if e.code != 404: raise return packages def get_latest_released_fedora_source_package_list(d): "Returns list of all the name os packages in the latest fedora distro" latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) return latest, package_names def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) return latest, package_names def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) return latest, package_names def get_latest_released_clear_source_package_list(d): latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) return latest, package_names def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" releases = [link.replace("Debian", "") for link in get_links_from_url(url, d) if link.startswith("Debian")] releases.sort() try: return releases[-1] except: return "_NotFound_" def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" import gzip package_names = set() for line in gzip.open(create_socket(url, d), mode="rt"): if line.startswith("Package:"): pkg = line.split(":", 1)[1].strip() package_names.add(pkg + ":" + section) return package_names def get_latest_released_debian_source_package_list(d): "Returns list of all the name of packages in the latest debian distro" latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" package_names = get_debian_style_source_package_list(url, "main", d) url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" package_names |= get_debian_style_source_package_list(url, "updates", d) return latest, package_names def find_latest_ubuntu_release(url, d): """ Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. To avoid matching development releases look for distributions that have updates, so the resulting distro could be any supported release. """ url += "?C=M;O=D" # Descending Sort by Last Modified for link in get_links_from_url(url, d): if "-updates" in link: distro = link.replace("-updates", "") return distro return "_NotFound_" def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest package_names = get_debian_style_source_package_list(url, "main", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest package_names |= get_debian_style_source_package_list(url, "updates", d) return latest, package_names def create_distro_packages_list(distro_check_dir, d): import shutil pkglst_dir = os.path.join(distro_check_dir, "package_lists") bb.utils.remove(pkglst_dir, True) bb.utils.mkdirhier(pkglst_dir) per_distro_functions = ( ("Debian", get_latest_released_debian_source_package_list), ("Ubuntu", get_latest_released_ubuntu_source_package_list), ("Fedora", get_latest_released_fedora_source_package_list), ("OpenSuSE", get_latest_released_opensuse_source_package_list), ("Mandriva", get_latest_released_mandriva_source_package_list), ("Clear", get_latest_released_clear_source_package_list), ) for name, fetcher_func in per_distro_functions: try: release, package_list = fetcher_func(d) except Exception as e: bb.warn("Cannot fetch packages for %s: %s" % (name, e)) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) if len(package_list) == 0: bb.error("Didn't fetch any packages for %s %s" % (name, release)) package_list_file = os.path.join(pkglst_dir, name + "-" + release) with open(package_list_file, 'w') as f: for pkg in sorted(package_list): f.write(pkg + "\n") def update_distro_data(distro_check_dir, datetime, d): """ If distro packages list data is old then rebuild it. The operations has to be protected by a lock so that only one thread performes it at a time. """ if not os.path.isdir (distro_check_dir): try: bb.note ("Making new directory: %s" % distro_check_dir) os.makedirs (distro_check_dir) except OSError: raise Exception('Unable to create directory %s' % (distro_check_dir)) datetime_file = os.path.join(distro_check_dir, "build_datetime") saved_datetime = "_invalid_" import fcntl try: if not os.path.exists(datetime_file): open(datetime_file, 'w+').close() # touch the file so that the next open won't fail f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) except OSError as e: raise Exception('Unable to open timestamp: %s' % e) finally: fcntl.lockf(f, fcntl.LOCK_UN) f.close() def compare_in_distro_packages_list(distro_check_dir, d): if not os.path.isdir(distro_check_dir): raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") localdata = bb.data.createCopy(d) pkglst_dir = os.path.join(distro_check_dir, "package_lists") matching_distros = [] pn = recipe_name = d.getVar('PN', True) bb.note("Checking: %s" % pn) if pn.find("-native") != -1: pnstripped = pn.split("-native") localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) bb.data.update_data(localdata) recipe_name = pnstripped[0] if pn.startswith("nativesdk-"): pnstripped = pn.split("nativesdk-") localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True)) bb.data.update_data(localdata) recipe_name = pnstripped[1] if pn.find("-cross") != -1: pnstripped = pn.split("-cross") localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) bb.data.update_data(localdata) recipe_name = pnstripped[0] if pn.find("-initial") != -1: pnstripped = pn.split("-initial") localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True)) bb.data.update_data(localdata) recipe_name = pnstripped[0] bb.note("Recipe: %s" % recipe_name) distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) tmp = localdata.getVar('DISTRO_PN_ALIAS', True) or "" for str in tmp.split(): if str and str.find("=") == -1 and distro_exceptions[str]: matching_distros.append(str) distro_pn_aliases = {} for str in tmp.split(): if "=" in str: (dist, pn_alias) = str.split('=') distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() for file in os.listdir(pkglst_dir): (distro, distro_release) = file.split("-") f = open(os.path.join(pkglst_dir, file), "r") for line in f: (pkg, section) = line.split(":") if distro.lower() in distro_pn_aliases: pn = distro_pn_aliases[distro.lower()] else: pn = recipe_name if pn == pkg: matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end f.close() break f.close() for item in tmp.split(): matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros def create_log_file(d, logname): logpath = d.getVar('LOG_DIR', True) bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname) logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix)) if not os.path.exists(logfile): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) open(logfile, 'w+').close() os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile def save_distro_check_result(result, datetime, result_file, d): pn = d.getVar('PN', True) logdir = d.getVar('LOG_DIR', True) if not logdir: bb.error("LOG_DIR variable is not defined, can't write the distro_check results") return bb.utils.mkdirhier(logdir) line = pn for i in result: line = line + "," + i f = open(result_file, "a") import fcntl fcntl.lockf(f, fcntl.LOCK_EX) f.seek(0, os.SEEK_END) # seek to the end of file f.write(line + "\n") fcntl.lockf(f, fcntl.LOCK_UN) f.close()