diff options
Diffstat (limited to 'meta/recipes-devtools/python/python3/create_manifest3.py')
| -rw-r--r-- | meta/recipes-devtools/python/python3/create_manifest3.py | 512 |
1 files changed, 301 insertions, 211 deletions
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py index 2f944f9b13..045240ea0b 100644 --- a/meta/recipes-devtools/python/python3/create_manifest3.py +++ b/meta/recipes-devtools/python/python3/create_manifest3.py @@ -22,7 +22,7 @@ # # # This way we will create a new manifest from the data structure that was built during -# this process, ont this new manifest each package will contain specifically only +# this process, on this new manifest each package will contain specifically only # what it needs to run. # # There are some caveats which we try to deal with, such as repeated files on different @@ -36,201 +36,269 @@ # Tha method to handle cached files does not work when a module includes a folder which # itself contains the pycache folder, gladly this is almost never the case. # -# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> +# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org> import sys import subprocess import json import os +import collections + +if '-d' in sys.argv: + debugFlag = '-d' +else: + debugFlag = '' + +# Get python version from ${PYTHON_MAJMIN} +pyversion = str(sys.argv[1]) # Hack to get native python search path (for folders), not fond of it but it works for now -pivot='recipe-sysroot-native' +pivot = 'recipe-sysroot-native' for p in sys.path: - if pivot in p: - nativelibfolder=p[:p.find(pivot)+len(pivot)] + if pivot in p: + nativelibfolder = p[:p.find(pivot)+len(pivot)] # Empty dict to hold the whole manifest -new_manifest = {} +new_manifest = collections.OrderedDict() # Check for repeated files, folders and wildcards -allfiles=[] -repeated=[] -wildcards=[] +allfiles = [] +repeated = [] +wildcards = [] -hasfolders=[] -allfolders=[] +hasfolders = [] +allfolders = [] def isFolder(value): - if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')): - return True - else: - return False + value = value.replace('${PYTHON_MAJMIN}',pyversion) + if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')): + return True + else: + return False def isCached(item): - if '__pycache__' in item: - return True - else: - return False + if '__pycache__' in item: + return True + else: + return False + +def prepend_comments(comments, json_manifest): + with open(json_manifest, 'r+') as manifest: + json_contents = manifest.read() + manifest.seek(0, 0) + manifest.write(comments + json_contents) + +def print_indent(msg, offset): + for l in msg.splitlines(): + msg = ' ' * offset + l + print(msg) + # Read existing JSON manifest with open('python3-manifest.json') as manifest: - old_manifest=json.load(manifest) - + # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker + manifest_str = manifest.read() + json_start = manifest_str.find('# EOC') + 6 # EOC + \n + manifest.seek(0) + comments = manifest.read(json_start) + manifest_str = manifest.read() + old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict) +# # First pass to get core-package functionality, because we base everything on the fact that core is actually working # Not exactly the same so it should not be a function -print ('Getting dependencies for package: core') - -# Special call to check for core package -output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') -for item in output.split(): - # We append it so it doesnt hurt what we currently have: - if isCached(item): - if item not in old_manifest['core']['cached']: - # We use the same data structure since its the one which will be used to check - # dependencies for other packages - old_manifest['core']['cached'].append(item) - else: - if item not in old_manifest['core']['files']: - # We use the same data structure since its the one which will be used to check - # dependencies for other packages - old_manifest['core']['files'].append(item) - -for value in old_manifest['core']['files']: - # Ignore folders, since we don't import those, difficult to handle multilib - if isFolder(value): - # Pass it directly - if isCached(value): - if value not in old_manifest['core']['cached']: - old_manifest['core']['cached'].append(value) - else: - if value not in old_manifest['core']['files']: - old_manifest['core']['files'].append(value) - continue - # Ignore binaries, since we don't import those, assume it was added correctly (manually) - if '${bindir}' in value: - # Pass it directly - if value not in old_manifest['core']['files']: - old_manifest['core']['files'].append(value) - continue - # Ignore empty values - if value == '': - continue - if '${includedir}' in value: - if value not in old_manifest['core']['files']: - old_manifest['core']['files'].append(value) - continue - # Get module name , shouldnt be affected by libdir/bindir - value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] - - # Launch separate task for each module for deterministic behavior - # Each module will only import what is necessary for it to work in specific - print ('Getting dependencies for module: %s' % value) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8') - print ('The following dependencies were found for module %s:\n' % value) - print (output) - for item in output.split(): - # We append it so it doesnt hurt what we currently have: - if isCached(item): - if item not in old_manifest['core']['cached']: - # We use the same data structure since its the one which will be used to check - # dependencies for other packages - old_manifest['core']['cached'].append(item) +# + +print_indent('Getting dependencies for package: core', 0) + + +# This special call gets the core dependencies and +# appends to the old manifest so it doesnt hurt what it +# currently holds. +# This way when other packages check for dependencies +# on the new core package, they will still find them +# even when checking the old_manifest + +output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8') +for coredep in output.split(): + coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') + if isCached(coredep): + if coredep not in old_manifest['core']['cached']: + old_manifest['core']['cached'].append(coredep) else: - if item not in old_manifest['core']['files']: - # We use the same data structure since its the one which will be used to check - # dependencies for other packages - old_manifest['core']['files'].append(item) - - -# We check which packages include folders -for key in old_manifest: - for value in old_manifest[key]['files']: - # Ignore folders, since we don't import those, difficult to handle multilib - if isFolder(value): - print ('%s is a folder' % value) - if key not in hasfolders: - hasfolders.append(key) - if value not in allfolders: - allfolders.append(value) - -for key in old_manifest: + if coredep not in old_manifest['core']['files']: + old_manifest['core']['files'].append(coredep) + + +# The second step is to loop through the existing files contained in the core package +# according to the old manifest, identify if they are modules, or some other type +# of file that we cant import (directories, binaries, configs) in which case we +# can only assume they were added correctly (manually) so we ignore those and +# pass them to the manifest directly. + +for filedep in old_manifest['core']['files']: + if isFolder(filedep): + if isCached(filedep): + if filedep not in old_manifest['core']['cached']: + old_manifest['core']['cached'].append(filedep) + else: + if filedep not in old_manifest['core']['files']: + old_manifest['core']['files'].append(filedep) + continue + if '${bindir}' in filedep: + if filedep not in old_manifest['core']['files']: + old_manifest['core']['files'].append(filedep) + continue + if filedep == '': + continue + if '${includedir}' in filedep: + if filedep not in old_manifest['core']['files']: + old_manifest['core']['files'].append(filedep) + continue + + # Get actual module name , shouldnt be affected by libdir/bindir, etc. + pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] + + # We now know that were dealing with a python module, so we can import it + # and check what its dependencies are. + # We launch a separate task for each module for deterministic behavior. + # Each module will only import what is necessary for it to work in specific. + # The output of each task will contain each module's dependencies + + print_indent('Getting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) + + + for pymodule_dep in output.split(): + pymodule_dep = pymodule_dep.replace(pyversion,'${PYTHON_MAJMIN}') + + if isCached(pymodule_dep): + if pymodule_dep not in old_manifest['core']['cached']: + old_manifest['core']['cached'].append(pymodule_dep) + else: + if pymodule_dep not in old_manifest['core']['files']: + old_manifest['core']['files'].append(pymodule_dep) + + +# At this point we are done with the core package. +# The old_manifest dictionary is updated only for the core package because +# all others will use this a base. + + +print('\n\nChecking for directories...\n') +# To improve the script speed, we check which packages contain directories +# since we will be looping through (only) those later. +for pypkg in old_manifest: + for filedep in old_manifest[pypkg]['files']: + if isFolder(filedep): + print_indent('%s is a directory' % filedep, 2) + if pypkg not in hasfolders: + hasfolders.append(pypkg) + if filedep not in allfolders: + allfolders.append(filedep) + + + +# This is the main loop that will handle each package. +# It works in a similar fashion than the step before, but +# we will now be updating a new dictionary that will eventually +# become the new manifest. +# +# The following loops though all packages in the manifest, +# through all files on each of them, and checks whether or not +# they are modules and can be imported. +# If they can be imported, then it checks for dependencies for +# each of them by launching a separate task. +# The output of that task is then parsed and the manifest is updated +# accordingly, wether it should add the module on FILES for the current package +# or if that module already belongs to another package then the current one +# will RDEPEND on it + +for pypkg in old_manifest: # Use an empty dict as data structure to hold data for each package and fill it up - new_manifest[key]={} - new_manifest[key]['files']=[] - - new_manifest[key]['rdepends']=[] + new_manifest[pypkg] = collections.OrderedDict() + new_manifest[pypkg]['summary'] = old_manifest[pypkg]['summary'] + new_manifest[pypkg]['rdepends'] = [] + new_manifest[pypkg]['files'] = [] + new_manifest[pypkg]['cached'] = old_manifest[pypkg]['cached'] + # All packages should depend on core - if key != 'core': - new_manifest[key]['rdepends'].append('core') - new_manifest[key]['cached']=[] - else: - new_manifest[key]['cached']=old_manifest[key]['cached'] - new_manifest[key]['summary']=old_manifest[key]['summary'] + if pypkg != 'core': + new_manifest[pypkg]['rdepends'].append('core') + new_manifest[pypkg]['cached'] = [] - # Handle special cases, we assume that when they were manually added - # to the manifest we knew what we were doing. print('\n') print('--------------------------') - print ('Handling package %s' % key) + print('Handling package %s' % pypkg) print('--------------------------') - special_packages=['misc', 'modules', 'dev'] - if key in special_packages or 'staticdev' in key: - print('Passing %s package directly' % key) - new_manifest[key]=old_manifest[key] + + # Handle special cases, we assume that when they were manually added + # to the manifest we knew what we were doing. + special_packages = ['misc', 'modules', 'dev', 'tests'] + if pypkg in special_packages or 'staticdev' in pypkg: + print_indent('Passing %s package directly' % pypkg, 2) + new_manifest[pypkg] = old_manifest[pypkg] continue - for value in old_manifest[key]['files']: - # We already handled core on the first pass - if key == 'core': - new_manifest[key]['files'].append(value) + for filedep in old_manifest[pypkg]['files']: + # We already handled core on the first pass, we can ignore it now + if pypkg == 'core': + if filedep not in new_manifest[pypkg]['files']: + new_manifest[pypkg]['files'].append(filedep) continue - # Ignore folders, since we don't import those, difficult to handle multilib - if isFolder(value): - # Pass folders directly - new_manifest[key]['files'].append(value) - # Ignore binaries, since we don't import those - if '${bindir}' in value: - # Pass it directly to the new manifest data structure - if value not in new_manifest[key]['files']: - new_manifest[key]['files'].append(value) + + # Handle/ignore what we cant import + if isFolder(filedep): + new_manifest[pypkg]['files'].append(filedep) + # Asyncio (and others) are both the package and the folder name, we should not skip those... + path,mod = os.path.split(filedep) + if mod != pypkg: + continue + if '${bindir}' in filedep: + if filedep not in new_manifest[pypkg]['files']: + new_manifest[pypkg]['files'].append(filedep) continue - # Ignore empty values - if value == '': + if filedep == '': continue - if '${includedir}' in value: - if value not in new_manifest[key]['files']: - new_manifest[key]['files'].append(value) + if '${includedir}' in filedep: + if filedep not in new_manifest[pypkg]['files']: + new_manifest[pypkg]['files'].append(filedep) continue - # Get module name , shouldnt be affected by libdir/bindir + # Get actual module name , shouldnt be affected by libdir/bindir, etc. # We need to check if the imported module comes from another (e.g. sqlite3.dump) - path,value = os.path.split(value) + path, pymodule = os.path.split(filedep) path = os.path.basename(path) - value = os.path.splitext(os.path.basename(value))[0] + pymodule = os.path.splitext(os.path.basename(pymodule))[0] # If this condition is met, it means we need to import it from another module # or its the folder itself (e.g. unittest) - if path == key: - if value: - value = path + '.' + value - else: - value = path - - # Launch separate task for each module for deterministic behavior - # Each module will only import what is necessary for it to work in specific - print ('\nGetting dependencies for module: %s' % value) - output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % value]).decode('utf8') - # We can print dependencies for debugging purposes - print ('The following dependencies were found for module %s:\n' % value) - print (output) - # Output will have all dependencies + if path == pypkg: + if pymodule: + pymodule = path + '.' + pymodule + else: + pymodule = path + + + + # We now know that were dealing with a python module, so we can import it + # and check what its dependencies are. + # We launch a separate task for each module for deterministic behavior. + # Each module will only import what is necessary for it to work in specific. + # The output of each task will contain each module's dependencies + + print_indent('\nGetting dependencies for module: %s' % pymodule, 2) + output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') + print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) + print_indent(output, 6) reportFILES = [] reportRDEPS = [] - for item in output.split(): + for pymodule_dep in output.split(): # Warning: This first part is ugly # One of the dependencies that was found, could be inside of one of the folders included by another package @@ -250,22 +318,28 @@ for key in old_manifest: # is folder_string inside path/folder1/folder2/filename?, # Yes, it works, but we waste a couple of milliseconds. - inFolders=False + pymodule_dep = pymodule_dep.replace(pyversion,'${PYTHON_MAJMIN}') + inFolders = False for folder in allfolders: - if folder in item: + # The module could have a directory named after it, e.g. xml, if we take out the filename from the path + # we'll end up with ${libdir}, and we want ${libdir}/xml + if isFolder(pymodule_dep): + check_path = pymodule_dep + else: + check_path = os.path.dirname(pymodule_dep) + if folder in check_path : inFolders = True # Did we find a folder? folderFound = False # Second flag to break inner for # Loop only through packages which contain folders - for keyfolder in hasfolders: + for pypkg_with_folder in hasfolders: if (folderFound == False): - #print('Checking folder %s on package %s' % (item,keyfolder)) - for file_folder in old_manifest[keyfolder]['files'] or file_folder in old_manifest[keyfolder]['cached']: - if file_folder==folder: - print ('%s folder found in %s' % (folder, keyfolder)) + # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) + for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: + if folder_dep == folder: + print ('%s directory found in %s' % (folder, pypkg_with_folder)) folderFound = True - if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key: - new_manifest[key]['rdepends'].append(keyfolder) - + if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: + new_manifest[pypkg]['rdepends'].append(pypkg_with_folder) else: break @@ -274,81 +348,97 @@ for key in old_manifest: continue - # We might already have it on the dictionary since it could depend on a (previously checked) module - if item not in new_manifest[key]['files'] and item not in new_manifest[key]['cached']: + + # No directories beyond this point + # We might already have this module on the dictionary since it could depend on a (previously checked) module + if pymodule_dep not in new_manifest[pypkg]['files'] and pymodule_dep not in new_manifest[pypkg]['cached']: # Handle core as a special package, we already did it so we pass it to NEW data structure directly - if key=='core': - print('Adding %s to %s FILES' % (item, key)) - if item.endswith('*'): - wildcards.append(item) - if isCached(item): - new_manifest[key]['cached'].append(item) - else: - new_manifest[key]['files'].append(item) - - # Check for repeated files - if item not in allfiles: - allfiles.append(item) - else: - repeated.append(item) + if pypkg == 'core': + print('Adding %s to %s FILES' % (pymodule_dep, pypkg)) + if pymodule_dep.endswith('*'): + wildcards.append(pymodule_dep) + if isCached(pymodule_dep): + new_manifest[pypkg]['cached'].append(pymodule_dep) + else: + new_manifest[pypkg]['files'].append(pymodule_dep) + # Check for repeated files + if pymodule_dep not in allfiles: + allfiles.append(pymodule_dep) + else: + if pymodule_dep not in repeated: + repeated.append(pymodule_dep) else: - # Check if this dependency is already contained on another package, so we add it + # Last step: Figure out if we this belongs to FILES or RDEPENDS + # We check if this module is already contained on another package, so we add that one # as an RDEPENDS, or if its not, it means it should be contained on the current - # package, so we should add it to FILES - for newkey in old_manifest: + # package, and we should add it to FILES + for possible_rdep in old_manifest: # Debug - #print('Checking %s ' % item + ' in %s' % newkey) - if item in old_manifest[newkey]['files'] or item in old_manifest[newkey]['cached']: - # Since were nesting, we need to check its not the same key - if(newkey!=key): - if newkey not in new_manifest[key]['rdepends']: - # Add it to the new manifest data struct - reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (newkey, key, item)) - new_manifest[key]['rdepends'].append(newkey) - break + # print('Checking %s ' % pymodule_dep + ' in %s' % possible_rdep) + if pymodule_dep in old_manifest[possible_rdep]['files'] or pymodule_dep in old_manifest[possible_rdep]['cached']: + # Since were nesting, we need to check its not the same pypkg + if(possible_rdep != pypkg): + if possible_rdep not in new_manifest[pypkg]['rdepends']: + # Add it to the new manifest data struct as RDEPENDS since it contains something this module needs + reportRDEPS.append('Adding %s to %s RDEPENDS, because it contains %s\n' % (possible_rdep, pypkg, pymodule_dep)) + new_manifest[pypkg]['rdepends'].append(possible_rdep) + break else: + + # Since this module wasnt found on another package, it is not an RDEP, + # so we add it to FILES for this package. # A module shouldn't contain itself (${libdir}/python3/sqlite3 shouldnt be on sqlite3 files) - if os.path.basename(item) != key: - reportFILES.append(('Adding %s to %s FILES\n' % (item, key))) - # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package - if isCached(item): - new_manifest[key]['cached'].append(item) + if os.path.basename(pymodule_dep) != pypkg: + reportFILES.append(('Adding %s to %s FILES\n' % (pymodule_dep, pypkg))) + if isCached(pymodule_dep): + new_manifest[pypkg]['cached'].append(pymodule_dep) else: - new_manifest[key]['files'].append(item) - - if item.endswith('*'): - wildcards.append(item) - if item not in allfiles: - allfiles.append(item) + new_manifest[pypkg]['files'].append(pymodule_dep) + if pymodule_dep.endswith('*'): + wildcards.append(pymodule_dep) + if pymodule_dep not in allfiles: + allfiles.append(pymodule_dep) else: - repeated.append(item) + if pymodule_dep not in repeated: + repeated.append(pymodule_dep) print('\n') print('#################################') - print('Summary for module %s' % value) - print('FILES found for module %s:' % value) + print('Summary for module %s' % pymodule) + print('FILES found for module %s:' % pymodule) print(''.join(reportFILES)) - print('RDEPENDS found for module %s:' % value) + print('RDEPENDS found for module %s:' % pymodule) print(''.join(reportRDEPS)) print('#################################') -print ('The following files are repeated (contained in more than one package), please check which package should get it:') -print (repeated) -print('The following files contain wildcards, please check they are necessary') +print('The following FILES contain wildcards, please check if they are necessary') print(wildcards) -print('The following files contain folders, please check they are necessary') +print('The following FILES contain folders, please check if they are necessary') print(hasfolders) + # Sort it just so it looks nicer -for key in new_manifest: - new_manifest[key]['files'].sort() - new_manifest[key]['cached'].sort() - new_manifest[key]['rdepends'].sort() +for pypkg in new_manifest: + new_manifest[pypkg]['files'].sort() + new_manifest[pypkg]['cached'].sort() + new_manifest[pypkg]['rdepends'].sort() # Create the manifest from the data structure that was built with open('python3-manifest.json.new','w') as outfile: - json.dump(new_manifest,outfile,sort_keys=True, indent=4) + json.dump(new_manifest,outfile, indent=4) outfile.write('\n') + +prepend_comments(comments,'python3-manifest.json.new') + +if (repeated): + error_msg = '\n\nERROR:\n' + error_msg += 'The following files were found in more than one package),\n' + error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' + error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' + error_msg += '\n'.join(repeated) + error_msg += '\n' + sys.exit(error_msg) + |