python3: Adds instructions to the manifest file

While there is a bit of documentation regarding building a new
manifest file for python, it seems that users usually only read
the manifest file.

The manifest file is in JSON format which doesn't allow comments,
hence why instructions were initially put elsewhere.

This patch hacks the call to open the JSON manifest file by using a
marker to trick it into reading only part of the file as the manifest
itself, and keep the other part as comments, which contain instructions
for the user to run the create_manifest task after an upgrade or when
adding a new package.

Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

4 files changed, 119 insertions, 7 deletions

diff --git a/meta/recipes-devtools/python/python3-native_3.5.6.bb b/meta/recipes-devtools/python/python3-native_3.5.6.bb
index 1e98c036c3..4c8faf88c0 100644
--- a/meta/recipes-devtools/python/python3-native_3.5.6.bb
+++ b/meta/recipes-devtools/python/python3-native_3.5.6.bb
@@ -85,7 +85,11 @@ python(){
     import json
     pythondir = d.getVar('THISDIR',True)
     with open(pythondir+'/python3/python3-manifest.json') as manifest_file:
-        python_manifest=json.load(manifest_file)
+        manifest_str =  manifest_file.read()
+        json_start = manifest_str.find('# EOC') + 6
+        manifest_file.seek(json_start)
+        manifest_str = manifest_file.read()
+        python_manifest = json.loads(manifest_str)
 
     rprovides = d.getVar('RPROVIDES').split()
 
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py
index fddb23cdc4..f7d4587030 100644
--- a/meta/recipes-devtools/python/python3/create_manifest3.py
+++ b/meta/recipes-devtools/python/python3/create_manifest3.py
@@ -22,7 +22,7 @@
 #
 #
 # This way we will create a new manifest from the data structure that was built during
-# this process, ont this new manifest each package will contain specifically only
+# this process, on this new manifest each package will contain specifically only
 # what it needs to run.
 #
 # There are some caveats which we try to deal with, such as repeated files on different
@@ -36,7 +36,7 @@
 # Tha method to handle cached files does not work when a module includes a folder which
 # itself contains the pycache folder, gladly this is almost never the case.
 #
-# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com>
+# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com>
 
 
 import sys
@@ -78,9 +78,21 @@ def isCached(item):
     else:
         return False
 
+def prepend_comments(comments, json_manifest):
+    with open(json_manifest, 'r+') as manifest:
+        json_contents = manifest.read()
+        manifest.seek(0, 0)
+        manifest.write(comments + json_contents)
+
 # Read existing JSON manifest
 with open('python3-manifest.json') as manifest:
-    old_manifest = json.load(manifest, object_pairs_hook=collections.OrderedDict)
+    # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
+    manifest_str =  manifest.read()
+    json_start = manifest_str.find('# EOC') + 6 # EOC + \n
+    manifest.seek(0)
+    comments = manifest.read(json_start)
+    manifest_str = manifest.read()
+    old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
 
 #
 # First pass to get core-package functionality, because we base everything on the fact that core is actually working
@@ -402,6 +414,8 @@ with open('python3-manifest.json.new','w') as outfile:
     json.dump(new_manifest,outfile, indent=4)
     outfile.write('\n')
 
+prepend_comments(comments,'python3-manifest.json.new')
+
 if (repeated):
     error_msg = '\n\nERROR:\n'
     error_msg += 'The following files are repeated (contained in more than one package),\n'
diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json
index 617256a3e0..3641d5bd7b 100644
--- a/meta/recipes-devtools/python/python3/python3-manifest.json
+++ b/meta/recipes-devtools/python/python3/python3-manifest.json
@@ -1,3 +1,93 @@
+# DO NOT (entirely) modify this file manually, please read.
+#
+# IMPORTANT NOTE:
+# Please keep in mind that the create_manifest task relies on the fact the the
+# target and native Python packages are the same, and it also needs to be executed
+# with a fully working native package (with all the PACKAGECONFIGs enabled and all
+# and all the modules should be working, check log.do_compile), otherwise the script
+# will fail to find dependencies correctly, this note is valid either if you are
+# upgrading to a new Python version or adding a new package.
+#
+#
+# If you are adding a new package please follow the next steps:
+#     How to add a new package:
+#     - If a user wants to add a new package all that has to be done is:
+#     Modify the python3-manifest.json file, and add the required file(s) to the FILES list,
+#     fill up the SUMMARY section as well, the script should handle all the rest.
+#
+#     Real example:
+#     We want to add a web browser package, including the file webbrowser.py
+#     which at the moment is on python3-misc.
+#     "webbrowser": {
+#         "files": ["${libdir}/python${PYTHON_MAJMIN}/lib-dynload/webbrowser.py"],
+#         "rdepends": [],
+#         "summary": "Python Web Browser support"}
+#
+#     * Note that the rdepends field was left empty
+#
+#     We run $ bitbake python3 -c create_manifest and the resulting manifest
+#     should be completed after a few seconds, showing something like:
+#     "webbrowser": {
+#         "files": ["${libdir}/python${PYTHON_MAJMIN}/webbrowser.py"],
+#         "rdepends": ["core","fcntl","io","pickle","shell","subprocess"],
+#         "summary": "Python Web Browser support"}
+#
+#
+# If you are upgrading Python to a new version please follow the next steps:
+#     After each Python upgrade, the create_manifest task should be executed, because we
+#     don't control what changes on upstream Python, so, some module dependency
+#     might have changed without us realizing it, a certain module can either have
+#     more or less dependencies, or could be depending on a new file that was just
+#     created on the new release and for obvious reasons we wouldn't have it on our
+#     old manifest, all of these issues would cause runtime errors on our system.
+#
+#     - Upgrade both the native and target Python packages to a new version
+#     - Run the create_manifest task for the target Python package as its shown below:
+#
+#     $ bitbake python3 -c create_manifest
+#
+#     This will automatically replace your manifest file located under the Python directory
+#     with an new one, which contains the new dependencies (if any).
+#
+#     Several things could have gone wrong here, I will try to explain a few:
+#
+#     a) A new file was introduced on this release, e.g. sha3*.so:
+#        The task will check what its needed to import every module, more than one module would
+#        would probably depend on sha3*.so, although only one module should contain it.
+#
+#        After running the task, the new manifest will have the sha3*.so file on more than one
+#        module, you need to manually decide which one of them should get it and delete it from
+#        the others, for example sha3*.so should likely be on ${PN}-crypt.
+#        Once you have deleted from the others you need to run the create_manifest task again,
+#        this will populate the other module's rdepends fields, with ${PN}-crypt and you should be
+#        good to go.
+#
+#     b) The native package wasn't built correctly and its missing a certain module:
+#        As mentioned before, you need to make sure the native package was built with all the modules
+#        because it is used as base to build the manifest file, you need to manually check log.do_compile
+#        since it won't error out the compile function if its only missing a couple of modules.
+#
+#        e.g. missing the _uuid module, log.do_compile would show the following:
+#        Python build finished successfully!
+#        The necessary bits to build these optional modules were not found:
+#        _uuid
+#
+#        What will happen here is that the new manifest would not be aware that the _uuid module exists, so
+#        not only we won't know of any dependencies to it, but also, the _uuid* files will be packaged on
+#        the misc package (which is where any file that doesn't belong anywhere else ends up).
+#
+#        This will eventually cause runtime errors on our system if we don't include the misc package on
+#        on our image, because the _uuid files will be missing.
+#        If we build the _uuid module correctly and run the create_manifest task the _uuid files will be
+#        detected correctly along with its dependencies, and we will get a working manifest.
+#
+#        This is the reason why it is important to make sure we have a fully working native build,
+#        so we can avoid these errors.
+#
+#
+#
+# DO NOT MODIFY THE NEXT LINE!, IT IS USED AS A MARKER FOR THE ACTUAL JSON MANIFEST
+# EOC
 {
     "tests": {
         "summary": "Python test suite",
@@ -1120,8 +1210,8 @@
             "unixadmin"
         ],
         "files": [
-            "${libdir}/python${PYTHON_MAJMIN}/venv",
-            "${bindir}/pyvenv*"
+            "${bindir}/pyvenv*",
+            "${libdir}/python${PYTHON_MAJMIN}/venv"
         ],
         "cached": []
     },
diff --git a/meta/recipes-devtools/python/python3_3.5.6.bb b/meta/recipes-devtools/python/python3_3.5.6.bb
index 25c47b6192..430c577785 100644
--- a/meta/recipes-devtools/python/python3_3.5.6.bb
+++ b/meta/recipes-devtools/python/python3_3.5.6.bb
@@ -265,7 +265,11 @@ python(){
     bb.parse.mark_dependency(d, filename)
 
     with open(filename) as manifest_file:
-        python_manifest=json.load(manifest_file, object_pairs_hook=collections.OrderedDict)
+        manifest_str =  manifest_file.read()
+        json_start = manifest_str.find('# EOC') + 6
+        manifest_file.seek(json_start)
+        manifest_str = manifest_file.read()
+        python_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
 
     include_pycs = d.getVar('INCLUDE_PYCS')