diff options
-rwxr-xr-x | bin/toaster | 23 | ||||
-rw-r--r-- | doc/bitbake-user-manual/bitbake-user-manual-fetching.xml | 8 | ||||
-rw-r--r-- | doc/bitbake-user-manual/bitbake-user-manual-metadata.xml | 2 | ||||
-rw-r--r-- | lib/bb/checksum.py | 2 | ||||
-rw-r--r-- | lib/bb/fetch2/__init__.py | 3 | ||||
-rw-r--r-- | lib/bb/fetch2/clearcase.py | 3 | ||||
-rw-r--r-- | lib/bb/fetch2/git.py | 2 | ||||
-rw-r--r-- | lib/bb/fetch2/npm.py | 1 | ||||
-rwxr-xr-x | lib/bb/main.py | 7 | ||||
-rw-r--r-- | lib/bb/providers.py | 6 | ||||
-rw-r--r-- | lib/bb/tests/fetch.py | 8 | ||||
-rw-r--r-- | lib/bb/utils.py | 13 | ||||
-rw-r--r-- | lib/toaster/bldcontrol/management/commands/checksettings.py | 5 | ||||
-rw-r--r-- | lib/toaster/toastergui/templates/base.html | 4 | ||||
-rw-r--r-- | lib/toaster/toastergui/templates/landing.html | 2 | ||||
-rwxr-xr-x | lib/toaster/toastergui/views.py | 57 | ||||
-rw-r--r-- | lib/toaster/toastergui/widgets.py | 4 |
17 files changed, 100 insertions, 50 deletions
diff --git a/bin/toaster b/bin/toaster index 762451df2..ed365ee82 100755 --- a/bin/toaster +++ b/bin/toaster @@ -18,9 +18,10 @@ # along with this program. If not, see http://www.gnu.org/licenses/. HELP=" -Usage: source toaster start|stop [webport=<address:port>] [noweb] +Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] Optional arguments: - [noweb] Setup the environment for building with toaster but don't start the development server + [nobuild] Setup the environment for capturing builds with toaster but disable managed builds + [noweb] Setup the environment for capturing builds with toaster but don't start the web server [webport] Set the development server (default: localhost:8000) " @@ -67,7 +68,7 @@ webserverKillAll() if [ -f ${pidfile} ]; then pid=`cat ${pidfile}` while kill -0 $pid 2>/dev/null; do - kill -SIGTERM -$pid 2>/dev/null + kill -SIGTERM $pid 2>/dev/null sleep 1 done rm ${pidfile} @@ -90,7 +91,7 @@ webserverStartAll() echo "Starting webserver..." - $MANAGE runserver "$ADDR_PORT" \ + $MANAGE runserver --noreload "$ADDR_PORT" \ </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \ & echo $! >${BUILDDIR}/.toastermain.pid @@ -183,6 +184,7 @@ unset OE_ROOT WEBSERVER=1 +export TOASTER_BUILDSERVER=1 ADDR_PORT="localhost:8000" unset CMD for param in $*; do @@ -190,6 +192,9 @@ for param in $*; do noweb ) WEBSERVER=0 ;; + nobuild ) + TOASTER_BUILDSERVER=0 + ;; start ) CMD=$param ;; @@ -286,9 +291,13 @@ case $CMD in return 4 fi export BITBAKE_UI='toasterui' - $MANAGE runbuilds \ - </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \ - & echo $! >${BUILDDIR}/.runbuilds.pid + if [ $TOASTER_BUILDSERVER -eq 1 ] ; then + $MANAGE runbuilds \ + </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \ + & echo $! >${BUILDDIR}/.runbuilds.pid + else + echo "Toaster build server not started." + fi # set fail safe stop system on terminal exit trap stop_system SIGHUP diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml index c721e86eb..43eae6b85 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml @@ -588,6 +588,14 @@ The name of the path in which to place the checkout. By default, the path is <filename>git/</filename>. </para></listitem> + <listitem><para><emphasis>"usehead":</emphasis> + Enables local <filename>git://</filename> URLs to use the + current branch HEAD as the revision for use with + <filename>AUTOREV</filename>. + The "usehead" parameter implies no branch and only works + when the transfer protocol is + <filename>file://</filename>. + </para></listitem> </itemizedlist> Here are some example URLs: <literallayout class='monospaced'> diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml index 0cfa53d02..918d0fbcb 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml @@ -502,7 +502,7 @@ </section> <section id='unsetting-variables'> - <title>Unseting variables</title> + <title>Unsetting variables</title> <para> It is possible to completely remove a variable or a variable flag diff --git a/lib/bb/checksum.py b/lib/bb/checksum.py index 84289208f..4e1598fe8 100644 --- a/lib/bb/checksum.py +++ b/lib/bb/checksum.py @@ -97,6 +97,8 @@ class FileChecksumCache(MultiProcessCache): def checksum_dir(pth): # Handle directories recursively + if pth == "/": + bb.fatal("Refusing to checksum /") dirchecksums = [] for root, dirs, files in os.walk(pth): for name in files: diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index f70f1b515..e758a6818 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -853,6 +853,9 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) + # Disable pseudo as it may affect ssh, potentially causing it to hang. + cmd = 'export PSEUDO_DISABLED=1; ' + cmd + logger.debug(1, "Running %s", cmd) success = False diff --git a/lib/bb/fetch2/clearcase.py b/lib/bb/fetch2/clearcase.py index 36beab6a5..3a6573d0b 100644 --- a/lib/bb/fetch2/clearcase.py +++ b/lib/bb/fetch2/clearcase.py @@ -69,7 +69,6 @@ from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError from bb.fetch2 import runfetchcmd from bb.fetch2 import logger -from distutils import spawn class ClearCase(FetchMethod): """Class to fetch urls via 'clearcase'""" @@ -107,7 +106,7 @@ class ClearCase(FetchMethod): else: ud.module = "" - ud.basecmd = d.getVar("FETCHCMD_ccrc") or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool") + ud.basecmd = d.getVar("FETCHCMD_ccrc") or "/usr/bin/env cleartool || rcleartool" if d.getVar("SRCREV") == "INVALID": raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.") diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 5ef8cd69e..758acb6a7 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -357,7 +357,7 @@ class Git(FetchMethod): logger.debug(1, "No Origin") runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py index b5f148ca0..ccc287b16 100644 --- a/lib/bb/fetch2/npm.py +++ b/lib/bb/fetch2/npm.py @@ -32,7 +32,6 @@ from bb.fetch2 import runfetchcmd from bb.fetch2 import logger from bb.fetch2 import UnpackError from bb.fetch2 import ParameterError -from distutils import spawn def subprocess_setup(): # Python installs a SIGPIPE handler by default. This is usually not what diff --git a/lib/bb/main.py b/lib/bb/main.py index 7711b290d..85d933266 100755 --- a/lib/bb/main.py +++ b/lib/bb/main.py @@ -401,9 +401,6 @@ def setup_bitbake(configParams, configuration, extrafeatures=None): # In status only mode there are no logs and no UI logger.addHandler(handler) - # Clear away any spurious environment variables while we stoke up the cooker - cleanedvars = bb.utils.clean_environment() - if configParams.server_only: featureset = [] ui_module = None @@ -419,6 +416,10 @@ def setup_bitbake(configParams, configuration, extrafeatures=None): server_connection = None + # Clear away any spurious environment variables while we stoke up the cooker + # (done after import_extension_module() above since for example import gi triggers env var usage) + cleanedvars = bb.utils.clean_environment() + if configParams.remote_server: # Connect to a remote XMLRPC server server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset, diff --git a/lib/bb/providers.py b/lib/bb/providers.py index 443187e17..c2aa98c06 100644 --- a/lib/bb/providers.py +++ b/lib/bb/providers.py @@ -244,17 +244,17 @@ def _filterProviders(providers, item, cfgData, dataCache): pkg_pn[pn] = [] pkg_pn[pn].append(p) - logger.debug(1, "providers for %s are: %s", item, list(pkg_pn.keys())) + logger.debug(1, "providers for %s are: %s", item, list(sorted(pkg_pn.keys()))) # First add PREFERRED_VERSIONS - for pn in pkg_pn: + for pn in sorted(pkg_pn): sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn) preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item) if preferred_versions[pn][1]: eligible.append(preferred_versions[pn][1]) # Now add latest versions - for pn in sortpkg_pn: + for pn in sorted(sortpkg_pn): if pn in preferred_versions and preferred_versions[pn][1]: continue preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0]) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 7d7c5d7ff..53146d903 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -757,12 +757,12 @@ class FetchLatestVersionTest(FetcherTest): ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") : "1.4.0", # combination version pattern - ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") + ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") : "1.2.0", ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") : "2014.01", # version pattern "yyyymmdd" - ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "") + ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "") : "20120614", # packages with a valid UPSTREAM_CHECK_GITTAGREGEX ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))") @@ -796,8 +796,8 @@ class FetchLatestVersionTest(FetcherTest): # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") : "2.0.0", - ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz") - : "6.1.19", + ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://ftp.debian.org/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") + : "5.3.10", } @skipIfNoNetwork() diff --git a/lib/bb/utils.py b/lib/bb/utils.py index c540b49cf..f75312399 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -523,12 +523,17 @@ def md5_file(filename): """ Return the hex string representation of the MD5 checksum of filename. """ - import hashlib - m = hashlib.md5() + import hashlib, mmap with open(filename, "rb") as f: - for line in f: - m.update(line) + m = hashlib.md5() + try: + with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: + for chunk in iter(lambda: mm.read(8192), b''): + m.update(chunk) + except ValueError: + # You can't mmap() an empty file so silence this exception + pass return m.hexdigest() def sha256_file(filename): diff --git a/lib/toaster/bldcontrol/management/commands/checksettings.py b/lib/toaster/bldcontrol/management/commands/checksettings.py index 582114ac9..823c6f154 100644 --- a/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -107,7 +107,10 @@ class Command(BaseCommand): action="ignore", message="^.*No fixture named.*$") print("Importing custom settings if present") - call_command("loaddata", "custom") + try: + call_command("loaddata", "custom") + except: + print("NOTE: optional fixture 'custom' not found") # we run lsupdates after config update print("\nFetching information from the layer index, " diff --git a/lib/toaster/toastergui/templates/base.html b/lib/toaster/toastergui/templates/base.html index edbd110c2..4f7206489 100644 --- a/lib/toaster/toastergui/templates/base.html +++ b/lib/toaster/toastergui/templates/base.html @@ -110,6 +110,7 @@ All builds </a> </li> + {% if project_enable %} <li id="navbar-all-projects" {% if request.resolver_match.url_name == 'all-projects' %} class="active" @@ -119,6 +120,7 @@ All projects </a> </li> + {% endif %} {% endif %} <li id="navbar-docs"> <a target="_blank" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html"> @@ -127,7 +129,9 @@ </a> </li> </ul> + {% if project_enable %} <a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a> + {% endif %} </div> </div> </nav> diff --git a/lib/toaster/toastergui/templates/landing.html b/lib/toaster/toastergui/templates/landing.html index cf7516dbc..70c7359fa 100644 --- a/lib/toaster/toastergui/templates/landing.html +++ b/lib/toaster/toastergui/templates/landing.html @@ -21,11 +21,13 @@ </p> {% if lvs_nos %} + {% if project_enable %} <p class="top-air"> <a class="btn btn-primary btn-lg" href="{% url 'newproject' %}"> Create your first Toaster project to run manage builds </a> </p> + {% endif %} {% else %} <div class="alert alert-info lead top-air"> Toaster has no layer information. Without layer information, you cannot run builds. To generate layer information you can: diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 209b07dad..34ed2b2e3 100755 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -49,6 +49,8 @@ import logging logger = logging.getLogger("toaster") +# Project creation and managed build enable +project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) class MimeTypeFinder(object): # setting this to False enables additional non-standard mimetypes @@ -65,6 +67,12 @@ class MimeTypeFinder(object): guessed_type = 'application/octet-stream' return guessed_type +# single point to add global values into the context before rendering +def toaster_render(request, page, context): + context['project_enable'] = project_enable + return render(request, page, context) + + # all new sessions should come through the landing page; # determine in which mode we are running in, and redirect appropriately def landing(request): @@ -86,7 +94,7 @@ def landing(request): context = {'lvs_nos' : Layer_Version.objects.all().count()} - return render(request, 'landing.html', context) + return toaster_render(request, 'landing.html', context) def objtojson(obj): from django.db.models.query import QuerySet @@ -519,7 +527,7 @@ def builddashboard( request, build_id ): 'packagecount' : packageCount, 'logmessages' : logmessages, } - return render( request, template, context ) + return toaster_render( request, template, context ) @@ -591,7 +599,7 @@ def task( request, build_id, task_id ): build__completed_on__lt=task_object.build.completed_on).exclude( order__isnull=True).exclude(outcome=Task.OUTCOME_NA).order_by('-build__completed_on') - return render( request, template, context ) + return toaster_render( request, template, context ) def recipe(request, build_id, recipe_id, active_tab="1"): template = "recipe.html" @@ -618,7 +626,7 @@ def recipe(request, build_id, recipe_id, active_tab="1"): 'package_count' : package_count, 'tab_states' : tab_states, } - return render(request, template, context) + return toaster_render(request, template, context) def recipe_packages(request, build_id, recipe_id): template = "recipe_packages.html" @@ -663,7 +671,7 @@ def recipe_packages(request, build_id, recipe_id): }, ] } - response = render(request, template, context) + response = toaster_render(request, template, context) _set_parameters_values(pagesize, orderby, request) return response @@ -785,7 +793,7 @@ def dirinfo(request, build_id, target_id, file_path=None): 'dir_list': dir_list, 'file_path': file_path, } - return render(request, template, context) + return toaster_render(request, template, context) def _find_task_dep(task_object): tdeps = Task_Dependency.objects.filter(task=task_object).filter(depends_on__order__gt=0) @@ -837,7 +845,7 @@ def configuration(request, build_id): 'build': build, 'project': build.project, 'targets': Target.objects.filter(build=build_id)}) - return render(request, template, context) + return toaster_render(request, template, context) def configvars(request, build_id): @@ -926,7 +934,7 @@ def configvars(request, build_id): ], } - response = render(request, template, context) + response = toaster_render(request, template, context) _set_parameters_values(pagesize, orderby, request) return response @@ -939,7 +947,7 @@ def bfile(request, build_id, package_id): 'project': build.project, 'objects' : files } - return render(request, template, context) + return toaster_render(request, template, context) # A set of dependency types valid for both included and built package views @@ -1092,7 +1100,7 @@ def package_built_detail(request, build_id, package_id): if paths.all().count() < 2: context['disable_sort'] = True; - response = render(request, template, context) + response = toaster_render(request, template, context) _set_parameters_values(pagesize, orderby, request) return response @@ -1111,7 +1119,7 @@ def package_built_dependencies(request, build_id, package_id): 'other_deps' : dependencies['other_deps'], 'dependency_count' : _get_package_dependency_count(package, -1, False) } - return render(request, template, context) + return toaster_render(request, template, context) def package_included_detail(request, build_id, target_id, package_id): @@ -1157,7 +1165,7 @@ def package_included_detail(request, build_id, target_id, package_id): } if paths.all().count() < 2: context['disable_sort'] = True - response = render(request, template, context) + response = toaster_render(request, template, context) _set_parameters_values(pagesize, orderby, request) return response @@ -1181,7 +1189,7 @@ def package_included_dependencies(request, build_id, target_id, package_id): 'reverse_count' : _get_package_reverse_dep_count(package, target_id), 'dependency_count' : _get_package_dependency_count(package, target_id, True) } - return render(request, template, context) + return toaster_render(request, template, context) def package_included_reverse_dependencies(request, build_id, target_id, package_id): template = "package_included_reverse_dependencies.html" @@ -1232,7 +1240,7 @@ def package_included_reverse_dependencies(request, build_id, target_id, package_ } if objects.all().count() < 2: context['disable_sort'] = True - response = render(request, template, context) + response = toaster_render(request, template, context) _set_parameters_values(pagesize, orderby, request) return response @@ -1365,6 +1373,9 @@ if True: # new project def newproject(request): + if not project_enable: + return redirect( landing ) + template = "newproject.html" context = { 'email': request.user.email if request.user.is_authenticated() else '', @@ -1379,7 +1390,7 @@ if True: if request.method == "GET": # render new project page - return render(request, template, context) + return toaster_render(request, template, context) elif request.method == "POST": mandatory_fields = ['projectname', 'ptype'] try: @@ -1419,7 +1430,7 @@ if True: context['alert'] = "Your chosen username is already used" else: context['alert'] = str(e) - return render(request, template, context) + return toaster_render(request, template, context) raise Exception("Invalid HTTP method for this page") @@ -1427,7 +1438,7 @@ if True: def project(request, pid): project = Project.objects.get(pk=pid) context = {"project": project} - return render(request, "project.html", context) + return toaster_render(request, "project.html", context) def jsunittests(request): """ Provides a page for the js unit tests """ @@ -1453,7 +1464,7 @@ if True: name="MACHINE", value="qemux86") context = {'project': new_project} - return render(request, "js-unit-tests.html", context) + return toaster_render(request, "js-unit-tests.html", context) from django.views.decorators.csrf import csrf_exempt @csrf_exempt @@ -1588,7 +1599,7 @@ if True: context = { 'project': Project.objects.get(id=pid), } - return render(request, template, context) + return toaster_render(request, template, context) def layerdetails(request, pid, layerid): project = Project.objects.get(pk=pid) @@ -1617,7 +1628,7 @@ if True: 'projectlayers': list(project_layers) } - return render(request, 'layerdetails.html', context) + return toaster_render(request, 'layerdetails.html', context) def get_project_configvars_context(): @@ -1707,7 +1718,7 @@ if True: except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist): pass - return render(request, "projectconf.html", context) + return toaster_render(request, "projectconf.html", context) def _file_names_for_artifact(build, artifact_type, artifact_id): """ @@ -1774,7 +1785,7 @@ if True: return response else: - return render(request, "unavailable_artifact.html") + return toaster_render(request, "unavailable_artifact.html") except (ObjectDoesNotExist, IOError): - return render(request, "unavailable_artifact.html") + return toaster_render(request, "unavailable_artifact.html") diff --git a/lib/toaster/toastergui/widgets.py b/lib/toaster/toastergui/widgets.py index 67c1ff961..a1792d997 100644 --- a/lib/toaster/toastergui/widgets.py +++ b/lib/toaster/toastergui/widgets.py @@ -41,6 +41,7 @@ import types import json import collections import re +import os from toastergui.tablefilter import TableFilterMap @@ -86,6 +87,9 @@ class ToasterTable(TemplateView): context['table_name'] = type(self).__name__.lower() context['empty_state'] = self.empty_state + # global variables + context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER')) + return context def get(self, request, *args, **kwargs): |