diff options
Diffstat (limited to 'meta/recipes-devtools/python/python3/create_manifest3.py')
| -rw-r--r-- | meta/recipes-devtools/python/python3/create_manifest3.py | 94 |
1 files changed, 62 insertions, 32 deletions
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py index 2db5e3b0b6..045240ea0b 100644 --- a/meta/recipes-devtools/python/python3/create_manifest3.py +++ b/meta/recipes-devtools/python/python3/create_manifest3.py @@ -22,7 +22,7 @@ # # # This way we will create a new manifest from the data structure that was built during -# this process, ont this new manifest each package will contain specifically only +# this process, on this new manifest each package will contain specifically only # what it needs to run. # # There are some caveats which we try to deal with, such as repeated files on different @@ -36,13 +36,19 @@ # Tha method to handle cached files does not work when a module includes a folder which # itself contains the pycache folder, gladly this is almost never the case. # -# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> +# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org> import sys import subprocess import json import os +import collections + +if '-d' in sys.argv: + debugFlag = '-d' +else: + debugFlag = '' # Get python version from ${PYTHON_MAJMIN} pyversion = str(sys.argv[1]) @@ -54,7 +60,7 @@ for p in sys.path: nativelibfolder = p[:p.find(pivot)+len(pivot)] # Empty dict to hold the whole manifest -new_manifest = {} +new_manifest = collections.OrderedDict() # Check for repeated files, folders and wildcards allfiles = [] @@ -77,16 +83,34 @@ def isCached(item): else: return False +def prepend_comments(comments, json_manifest): + with open(json_manifest, 'r+') as manifest: + json_contents = manifest.read() + manifest.seek(0, 0) + manifest.write(comments + json_contents) + +def print_indent(msg, offset): + for l in msg.splitlines(): + msg = ' ' * offset + l + print(msg) + + # Read existing JSON manifest with open('python3-manifest.json') as manifest: - old_manifest = json.load(manifest) + # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker + manifest_str = manifest.read() + json_start = manifest_str.find('# EOC') + 6 # EOC + \n + manifest.seek(0) + comments = manifest.read(json_start) + manifest_str = manifest.read() + old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict) # # First pass to get core-package functionality, because we base everything on the fact that core is actually working # Not exactly the same so it should not be a function # -print ('Getting dependencies for package: core') +print_indent('Getting dependencies for package: core', 0) # This special call gets the core dependencies and @@ -96,7 +120,7 @@ print ('Getting dependencies for package: core') # on the new core package, they will still find them # even when checking the old_manifest -output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') +output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8') for coredep in output.split(): coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') if isCached(coredep): @@ -136,17 +160,16 @@ for filedep in old_manifest['core']['files']: # Get actual module name , shouldnt be affected by libdir/bindir, etc. pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] - # We now know that were dealing with a python module, so we can import it # and check what its dependencies are. # We launch a separate task for each module for deterministic behavior. # Each module will only import what is necessary for it to work in specific. # The output of each task will contain each module's dependencies - print ('Getting dependencies for module: %s' % pymodule) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') - print ('The following dependencies were found for module %s:\n' % pymodule) - print (output) + print_indent('Getting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) for pymodule_dep in output.split(): @@ -165,12 +188,13 @@ for filedep in old_manifest['core']['files']: # all others will use this a base. +print('\n\nChecking for directories...\n') # To improve the script speed, we check which packages contain directories # since we will be looping through (only) those later. for pypkg in old_manifest: for filedep in old_manifest[pypkg]['files']: if isFolder(filedep): - print ('%s is a folder' % filedep) + print_indent('%s is a directory' % filedep, 2) if pypkg not in hasfolders: hasfolders.append(pypkg) if filedep not in allfolders: @@ -195,29 +219,27 @@ for pypkg in old_manifest: for pypkg in old_manifest: # Use an empty dict as data structure to hold data for each package and fill it up - new_manifest[pypkg] = {} - new_manifest[pypkg]['files'] = [] + new_manifest[pypkg] = collections.OrderedDict() + new_manifest[pypkg]['summary'] = old_manifest[pypkg]['summary'] new_manifest[pypkg]['rdepends'] = [] + new_manifest[pypkg]['files'] = [] + new_manifest[pypkg]['cached'] = old_manifest[pypkg]['cached'] # All packages should depend on core if pypkg != 'core': - new_manifest[pypkg]['rdepends'].append('core') - new_manifest[pypkg]['cached'] = [] - else: - new_manifest[pypkg]['cached'] = old_manifest[pypkg]['cached'] - new_manifest[pypkg]['summary'] = old_manifest[pypkg]['summary'] - + new_manifest[pypkg]['rdepends'].append('core') + new_manifest[pypkg]['cached'] = [] print('\n') print('--------------------------') - print ('Handling package %s' % pypkg) + print('Handling package %s' % pypkg) print('--------------------------') # Handle special cases, we assume that when they were manually added # to the manifest we knew what we were doing. - special_packages = ['misc', 'modules', 'dev', 'tests', 'sqlite3-tests'] + special_packages = ['misc', 'modules', 'dev', 'tests'] if pypkg in special_packages or 'staticdev' in pypkg: - print('Passing %s package directly' % pypkg) + print_indent('Passing %s package directly' % pypkg, 2) new_manifest[pypkg] = old_manifest[pypkg] continue @@ -248,7 +270,7 @@ for pypkg in old_manifest: # Get actual module name , shouldnt be affected by libdir/bindir, etc. # We need to check if the imported module comes from another (e.g. sqlite3.dump) - path,pymodule = os.path.split(filedep) + path, pymodule = os.path.split(filedep) path = os.path.basename(path) pymodule = os.path.splitext(os.path.basename(pymodule))[0] @@ -268,10 +290,10 @@ for pypkg in old_manifest: # Each module will only import what is necessary for it to work in specific. # The output of each task will contain each module's dependencies - print ('\nGetting dependencies for module: %s' % pymodule) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') - print ('The following dependencies were found for module %s:\n' % pymodule) - print (output) + print_indent('\nGetting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) reportFILES = [] reportRDEPS = [] @@ -299,7 +321,13 @@ for pypkg in old_manifest: pymodule_dep = pymodule_dep.replace(pyversion,'${PYTHON_MAJMIN}') inFolders = False for folder in allfolders: - if folder in pymodule_dep: + # The module could have a directory named after it, e.g. xml, if we take out the filename from the path + # we'll end up with ${libdir}, and we want ${libdir}/xml + if isFolder(pymodule_dep): + check_path = pymodule_dep + else: + check_path = os.path.dirname(pymodule_dep) + if folder in check_path : inFolders = True # Did we find a folder? folderFound = False # Second flag to break inner for # Loop only through packages which contain folders @@ -308,7 +336,7 @@ for pypkg in old_manifest: # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: if folder_dep == folder: - print ('%s folder found in %s' % (folder, pypkg_with_folder)) + print ('%s directory found in %s' % (folder, pypkg_with_folder)) folderFound = True if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: new_manifest[pypkg]['rdepends'].append(pypkg_with_folder) @@ -400,12 +428,14 @@ for pypkg in new_manifest: # Create the manifest from the data structure that was built with open('python3-manifest.json.new','w') as outfile: - json.dump(new_manifest,outfile,sort_keys=True, indent=4) + json.dump(new_manifest,outfile, indent=4) outfile.write('\n') +prepend_comments(comments,'python3-manifest.json.new') + if (repeated): error_msg = '\n\nERROR:\n' - error_msg += 'The following files are repeated (contained in more than one package),\n' + error_msg += 'The following files were found in more than one package),\n' error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' error_msg += '\n'.join(repeated) |