diff options
Diffstat (limited to 'lib/toaster/toastermain/management/commands/buildimport.py')
| -rw-r--r-- | lib/toaster/toastermain/management/commands/buildimport.py | 579 |
1 files changed, 579 insertions, 0 deletions
diff --git a/lib/toaster/toastermain/management/commands/buildimport.py b/lib/toaster/toastermain/management/commands/buildimport.py new file mode 100644 index 000000000..f7139aa04 --- /dev/null +++ b/lib/toaster/toastermain/management/commands/buildimport.py @@ -0,0 +1,579 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2018 Wind River Systems +# +# SPDX-License-Identifier: GPL-2.0-only +# + +# buildimport: import a project for project specific configuration +# +# Usage: +# (a) Set up Toaster environent +# +# (b) Call buildimport +# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ +# --name=$PROJECTNAME \ +# --path=$BUILD_DIRECTORY \ +# --callback="$CALLBACK_SCRIPT" \ +# --command="configure|reconfigure|import" +# +# (c) Return is "|Default_image=%s|Project_id=%d" +# +# (d) Open Toaster to this project using for example: +# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id +# +# (e) To delete a project: +# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ +# --name=$PROJECTNAME --delete-project +# + + +# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import + +from django.core.management.base import BaseCommand +from orm.models import Project, Release, ProjectVariable +from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer +from toastergui.api import scan_layer_content + +import os +import re +import os.path +import subprocess +import shutil + +# Toaster variable section delimiters +TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ===' +TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ===' + +# quick development/debugging support +verbose = 2 +def _log(msg): + if 1 == verbose: + print(msg) + elif 2 == verbose: + f1=open('/tmp/toaster.log', 'a') + f1.write("|" + msg + "|\n" ) + f1.close() + + +__config_regexp__ = re.compile( r""" + ^ + (?P<exp>export\s+)? + (?P<var>[a-zA-Z0-9\-_+.${}/~]+?) + (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])? + + \s* ( + (?P<colon>:=) | + (?P<lazyques>\?\?=) | + (?P<ques>\?=) | + (?P<append>\+=) | + (?P<prepend>=\+) | + (?P<predot>=\.) | + (?P<postdot>\.=) | + = + ) \s* + + (?!'[^']*'[^']*'$) + (?!\"[^\"]*\"[^\"]*\"$) + (?P<apo>['\"]) + (?P<value>.*) + (?P=apo) + $ + """, re.X) + +class Command(BaseCommand): + args = "<name> <path> <release>" + help = "Import a command line build directory" + vars = {} + toaster_vars = {} + + def add_arguments(self, parser): + parser.add_argument( + '--name', dest='name', required=True, + help='name of the project', + ) + parser.add_argument( + '--path', dest='path', required=True, + help='path to the project', + ) + parser.add_argument( + '--release', dest='release', required=False, + help='release for the project', + ) + parser.add_argument( + '--callback', dest='callback', required=False, + help='callback for project config update', + ) + parser.add_argument( + '--delete-project', dest='delete_project', required=False, + help='delete this project from the database', + ) + parser.add_argument( + '--command', dest='command', required=False, + help='command (configure,reconfigure,import)', + ) + + def get_var(self, varname): + value = self.vars.get(varname, '') + if value: + varrefs = re.findall('\${([^}]*)}', value) + for ref in varrefs: + if ref in self.vars: + value = value.replace('${%s}' % ref, self.vars[ref]) + return value + + # Extract the bb variables from a conf file + def scan_conf(self,fn): + vars = self.vars + toaster_vars = self.toaster_vars + + #_log("scan_conf:%s" % fn) + if not os.path.isfile(fn): + return + f = open(fn, 'r') + + #statements = ast.StatementGroup() + lineno = 0 + is_toaster_section = False + while True: + lineno = lineno + 1 + s = f.readline() + if not s: + break + w = s.strip() + # skip empty lines + if not w: + continue + # evaluate Toaster sections + if w.startswith(TOASTER_PROLOG): + is_toaster_section = True + continue + if w.startswith(TOASTER_EPILOG): + is_toaster_section = False + continue + s = s.rstrip() + while s[-1] == '\\': + s2 = f.readline().strip() + lineno = lineno + 1 + if (not s2 or s2 and s2[0] != "#") and s[0] == "#" : + echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + s = s[:-1] + s2 + # skip comments + if s[0] == '#': + continue + # process the line for just assignments + m = __config_regexp__.match(s) + if m: + groupd = m.groupdict() + var = groupd['var'] + value = groupd['value'] + + if groupd['lazyques']: + if not var in vars: + vars[var] = value + continue + if groupd['ques']: + if not var in vars: + vars[var] = value + continue + # preset empty blank for remaining operators + if not var in vars: + vars[var] = '' + if groupd['append']: + vars[var] += value + elif groupd['prepend']: + vars[var] = "%s%s" % (value,vars[var]) + elif groupd['predot']: + vars[var] = "%s %s" % (value,vars[var]) + elif groupd['postdot']: + vars[var] = "%s %s" % (vars[var],value) + else: + vars[var] = "%s" % (value) + # capture vars in a Toaster section + if is_toaster_section: + toaster_vars[var] = vars[var] + + # DONE WITH PARSING + f.close() + self.vars = vars + self.toaster_vars = toaster_vars + + # Update the scanned project variables + def update_project_vars(self,project,name): + pv, create = ProjectVariable.objects.get_or_create(project = project, name = name) + if (not name in self.vars.keys()) or (not self.vars[name]): + self.vars[name] = pv.value + else: + if pv.value != self.vars[name]: + pv.value = self.vars[name] + pv.save() + + # Find the git version of the installation + def find_layer_dir_version(self,path): + # * rocko ... + + install_version = '' + cwd = os.getcwd() + os.chdir(path) + p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + out = out.decode("utf-8") + for branch in out.split('\n'): + if ('*' == branch[0:1]) and ('no branch' not in branch): + install_version = re.sub(' .*','',branch[2:]) + break + if 'remotes/m/master' in branch: + install_version = re.sub('.*base/','',branch) + break + os.chdir(cwd) + return install_version + + # Compute table of the installation's registered layer versions (branch or commit) + def find_layer_dir_versions(self,INSTALL_URL_PREFIX): + lv_dict = {} + layer_versions = Layer_Version.objects.all() + for lv in layer_versions: + layer = Layer.objects.filter(pk=lv.layer.pk)[0] + if layer.vcs_url: + url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'') + else: + url_short = '' + # register the core, branch, and the version variations + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name) + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name) + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name) + #_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name)) + return lv_dict + + # Apply table of all layer versions + def extract_bblayers(self): + # set up the constants + bblayer_str = self.get_var('BBLAYERS') + TOASTER_DIR = os.environ.get('TOASTER_DIR') + INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/" + TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/" + INSTALL_URL_PREFIX = '' + layers = Layer.objects.filter(name='openembedded-core') + for layer in layers: + if layer.vcs_url: + INSTALL_URL_PREFIX = layer.vcs_url + break + INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/") + INSTALL_VERSION_DIR = TOASTER_DIR + INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_') + INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_') + INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX) + + # get the set of available layer:layer_versions + lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX) + + # compute the layer matches + layers_list = [] + for line in bblayer_str.split(' '): + if not line: + continue + if line.endswith('/local'): + continue + + # isolate the repo + layer_path = line + line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/') + + # isolate the sub-path + path_index = line.rfind('/') + if path_index > 0: + sub_path = line[path_index+1:] + line = line[0:path_index] + else: + sub_path = '' + + # isolate the version + if TOASTER_CLONE_PREFIX in layer_path: + is_toaster_clone = True + # extract version from name syntax + version_index = line.find('_') + if version_index > 0: + version = line[version_index+1:] + line = line[0:version_index] + else: + version = '' + _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version)) + else: + is_toaster_clone = False + # version is from the installation + version = self.find_layer_dir_version(layer_path) + _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version)) + + # capture the layer information into layers_list + layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) ) + return layers_list,lv_dict + + # + def find_import_release(self,layers_list,lv_dict,default_release): + # poky,meta,rocko => 4;openembedded-core + release = default_release + for line,path,version,layer_path,is_toaster_clone in layers_list: + key = "%s,%s,%s" % (line,path,version) + if key in lv_dict: + lv_id = lv_dict[key] + if 'openembedded-core' == lv_id[1]: + _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone)) + # only versions in Toaster managed layers are accepted + if not is_toaster_clone: + break + try: + release = Release.objects.get(name=version) + except: + pass + break + _log("Find_import_release:RELEASE=%s" % release.name) + return release + + # Apply the found conf layers + def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None): + for line,path,version,layer_path,is_toaster_clone in layers_list: + # Assert release promote if present + if release: + version = release + # try to match the key to a layer_version + key = "%s,%s,%s" % (line,path,version) + key_short = "%s,%s,%s" % (line,path,'') + lv_id = '' + if key in lv_dict: + lv_id = lv_dict[key] + lv = Layer_Version.objects.get(pk=int(lv_id[0])) + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=lv) + pl.optional=False + pl.save() + _log(" %s => %s;%s" % (key,lv_id[0],lv_id[1])) + elif key_short in lv_dict: + lv_id = lv_dict[key_short] + lv = Layer_Version.objects.get(pk=int(lv_id[0])) + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=lv) + pl.optional=False + pl.save() + _log(" %s ?> %s" % (key,lv_dict[key_short])) + else: + _log("%s <= %s" % (key,layer_path)) + found = False + # does local layer already exist in this project? + try: + for pl in ProjectLayer.objects.filter(project=project): + if pl.layercommit.layer.local_source_dir == layer_path: + found = True + _log(" Project Local Layer found!") + except Exception as e: + _log("ERROR: Local Layer '%s'" % e) + pass + + if not found: + # Does Layer name+path already exist? + try: + layer_name_base = os.path.basename(layer_path) + _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path)) + layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path) + # Found! Attach layer_version and ProjectLayer + layer_version = Layer_Version.objects.create( + layer=layer, + project=project, + layer_source=LayerSource.TYPE_IMPORTED) + layer_version.save() + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=layer_version) + pl.optional=False + pl.save() + found = True + # add layer contents to this layer version + scan_layer_content(layer,layer_version) + _log(" Parent Local Layer found in db!") + except Exception as e: + _log("Layer_exists_test_failed: Local Layer '%s'" % e) + pass + + if not found: + # Insure that layer path exists, in case of user typo + if not os.path.isdir(layer_path): + _log("ERROR:Layer path '%s' not found" % layer_path) + continue + # Add layer to db and attach project to it + layer_name_base = os.path.basename(layer_path) + # generate a unique layer name + layer_name_matches = {} + for layer in Layer.objects.filter(name__contains=layer_name_base): + layer_name_matches[layer.name] = '1' + layer_name_idx = 0 + layer_name_test = layer_name_base + while layer_name_test in layer_name_matches.keys(): + layer_name_idx += 1 + layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx) + # create the layer and layer_verion objects + layer = Layer.objects.create(name=layer_name_test) + layer.local_source_dir = layer_path + layer_version = Layer_Version.objects.create( + layer=layer, + project=project, + layer_source=LayerSource.TYPE_IMPORTED) + layer.save() + layer_version.save() + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=layer_version) + pl.optional=False + pl.save() + # register the layer's content + _log(" Local Layer Add content") + scan_layer_content(layer,layer_version) + _log(" Local Layer Added '%s'!" % layer_name_test) + + # Scan the project's conf files (if any) + def scan_conf_variables(self,project_path): + self.vars['TOPDIR'] = project_path + # scan the project's settings, add any new layers or variables + if os.path.isfile("%s/conf/local.conf" % project_path): + self.scan_conf("%s/conf/local.conf" % project_path) + self.scan_conf("%s/conf/bblayers.conf" % project_path) + # Import then disable old style Toaster conf files (before 'merged_attr') + old_toaster_local = "%s/conf/toaster.conf" % project_path + if os.path.isfile(old_toaster_local): + self.scan_conf(old_toaster_local) + shutil.move(old_toaster_local, old_toaster_local+"_old") + old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path + if os.path.isfile(old_toaster_layer): + self.scan_conf(old_toaster_layer) + shutil.move(old_toaster_layer, old_toaster_layer+"_old") + + # Scan the found conf variables (if any) + def apply_conf_variables(self,project,layers_list,lv_dict,release=None): + if self.vars: + # Catch vars relevant to Toaster (in case no Toaster section) + self.update_project_vars(project,'DISTRO') + self.update_project_vars(project,'MACHINE') + self.update_project_vars(project,'IMAGE_INSTALL:append') + self.update_project_vars(project,'IMAGE_FSTYPES') + self.update_project_vars(project,'PACKAGE_CLASSES') + # These vars are typically only assigned by Toaster + #self.update_project_vars(project,'DL_DIR') + #self.update_project_vars(project,'SSTATE_DIR') + + # Assert found Toaster vars + for var in self.toaster_vars.keys(): + pv, create = ProjectVariable.objects.get_or_create(project = project, name = var) + pv.value = self.toaster_vars[var] + _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value)) + pv.save() + + # Assert found BBLAYERS + if 0 < verbose: + for pl in ProjectLayer.objects.filter(project=project): + release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name + print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) + self.apply_conf_bblayers(layers_list,lv_dict,project,release) + if 0 < verbose: + for pl in ProjectLayer.objects.filter(project=project): + release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name + print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) + + def handle(self, *args, **options): + project_name = options['name'] + project_path = options['path'] + project_callback = options['callback'] if options['callback'] else '' + release_name = options['release'] if options['release'] else '' + + # + # Delete project + # + + if options['delete_project']: + try: + print("Project '%s' delete from Toaster database" % (project_name)) + project = Project.objects.get(name=project_name) + # TODO: deep project delete + project.delete() + print("Project '%s' Deleted" % (project_name)) + return + except Exception as e: + print("Project '%s' not found, not deleted (%s)" % (project_name,e)) + return + + # + # Create/Update/Import project + # + + # See if project (by name) exists + project = None + try: + # Project already exists + project = Project.objects.get(name=project_name) + except Exception as e: + pass + + # Find the installation's default release + default_release = Release.objects.get(id=1) + + # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import' + if ("reconfigure" == options['command']) and project is None: + options['command'] = 'import' + + # 'Configure': + if "configure" == options['command']: + # Note: ignore any existing conf files + # create project, SANITY: reuse any project of same name + project = Project.objects.create_project(project_name,default_release,project) + + # 'Re-configure': + if "reconfigure" == options['command']: + # Scan the directory's conf files + self.scan_conf_variables(project_path) + # Scan the layer list + layers_list,lv_dict = self.extract_bblayers() + # Apply any new layers or variables + self.apply_conf_variables(project,layers_list,lv_dict) + + # 'Import': + if "import" == options['command']: + # Scan the directory's conf files + self.scan_conf_variables(project_path) + # Remove these Toaster controlled variables + for var in ('DL_DIR','SSTATE_DIR'): + self.vars.pop(var, None) + self.toaster_vars.pop(var, None) + # Scan the layer list + layers_list,lv_dict = self.extract_bblayers() + # Find the directory's release, and promote to default_release if local paths + release = self.find_import_release(layers_list,lv_dict,default_release) + # create project, SANITY: reuse any project of same name + project = Project.objects.create_project(project_name,release,project, imported=True) + # Apply any new layers or variables + self.apply_conf_variables(project,layers_list,lv_dict,release) + # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific' + project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1') + + # Set up the project's meta data + project.builddir = project_path + project.merged_attr = True + project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback) + project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT) + if ("configure" == options['command']) or ("import" == options['command']): + # preset the mode and default image recipe + project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW) + project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal") + + # Assert any extended/custom actions or variables for new non-Toaster projects + if not len(self.toaster_vars): + pass + else: + project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE) + + # Save the updated Project + project.save() + + _log("Buildimport:project='%s' at '%d'" % (project_name,project.id)) + + if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']): + print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id)) + else: + print("|Project_id=%d" % (project.id)) + |