lib/bb/utils: fix and extend edit_metadata_file()

Fix several bugs and add some useful enhancements to make this into a
more generic metadata editing function:
* Support modifying function values (name must be specified ending with
  "()")
* Support dropping values by returning None as the new value
* Split out edit_metadata() function to provide same functionality
  on a list/iterable
* Pass operation to callback and allow function to return them
* Pass current output lines to callback so they can be modified
* Fix handling of single-quoted values
* Handle :=, =+, .=, and =. operators
* Support arbitrary indent string
* Support indenting by length of assignment (by specifying -1)
* Fix typo in variablename - intentspc -> indentspc
* Expand function docstring to cover arguments / usage
* Add a parameter to enable matching names with overrides applied
* Add some bitbake-selftest tests

Note that this does change the expected signature of the callback
function. The only known caller is in lib/bb/utils.py itself; I doubt
anyone else has made extensive use of this function yet.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 271 insertions, 0 deletions

diff --git a/lib/bb/tests/utils.py b/lib/bb/tests/utils.py
index 6e09858e5..9171509a6 100644
--- a/lib/bb/tests/utils.py
+++ b/lib/bb/tests/utils.py
@@ -22,6 +22,7 @@
 import unittest
 import bb
 import os
+import tempfile
 
 class VerCmpString(unittest.TestCase):
 
@@ -105,3 +106,273 @@ class Path(unittest.TestCase):
         for arg1, correctresult in checkitems:
             result = bb.utils._check_unsafe_delete_path(arg1)
             self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
+
+
+class EditMetadataFile(unittest.TestCase):
+    _origfile = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+    def _testeditfile(self, varvalues, compareto, dummyvars=None):
+        if dummyvars is None:
+            dummyvars = []
+        with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+            tf.write(self._origfile)
+            tf.close()
+            try:
+                varcalls = []
+                def handle_file(varname, origvalue, op, newlines):
+                    self.assertIn(varname, varvalues, 'Callback called for variable %s not in the list!' % varname)
+                    self.assertNotIn(varname, dummyvars, 'Callback called for variable %s in dummy list!' % varname)
+                    varcalls.append(varname)
+                    return varvalues[varname]
+
+                bb.utils.edit_metadata_file(tf.name, varvalues.keys(), handle_file)
+                with open(tf.name) as f:
+                    modfile = f.readlines()
+                # Ensure the output matches the expected output
+                self.assertEqual(compareto.splitlines(True), modfile)
+                # Ensure the callback function was called for every variable we asked for
+                # (plus allow testing behaviour when a requested variable is not present)
+                self.assertEqual(sorted(varvalues.keys()), sorted(varcalls + dummyvars))
+            finally:
+                os.remove(tf.name)
+
+
+    def test_edit_metadata_file_nochange(self):
+        # Test file doesn't get modified with nothing to do
+        self._testeditfile({}, self._origfile)
+        # Test file doesn't get modified with only dummy variables
+        self._testeditfile({'DUMMY1': ('should_not_set', None, 0, True),
+                        'DUMMY2': ('should_not_set_again', None, 0, True)}, self._origfile, dummyvars=['DUMMY1', 'DUMMY2'])
+        # Test file doesn't get modified with some the same values
+        self._testeditfile({'THIS': ('that', None, 0, True),
+                        'OTHER': ('anothervalue', None, 0, True),
+                        'MULTILINE3': ('               c1               c2               c3', None, 4, False)}, self._origfile)
+
+    def test_edit_metadata_file_1(self):
+
+        newfile1 = """
+# A comment
+HELLO = "newvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+        self._testeditfile({'HELLO': ('newvalue', None, 4, True)}, newfile1)
+
+
+    def test_edit_metadata_file_2(self):
+
+        newfile2 = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = " \\
+    d1 \\
+    d2 \\
+    d3 \\
+    "
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = "nowsingle"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+        self._testeditfile({'MULTILINE': (['d1','d2','d3'], None, 4, False),
+                        'MULTILINE3': ('nowsingle', None, 4, True),
+                        'NOTPRESENT': (['a', 'b'], None, 4, False)}, newfile2, dummyvars=['NOTPRESENT'])
+
+
+    def test_edit_metadata_file_3(self):
+
+        newfile3 = """
+# A comment
+HELLO = "oldvalue"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = "yetanothervalue"
+
+MULTILINE = "e1 \\
+             e2 \\
+             e3 \\
+             "
+
+MULTILINE2 := "f1 \\
+\tf2 \\
+\t"
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    othercommand_one a b c
+    othercommand_two d e f
+}
+"""
+
+        self._testeditfile({'do_functionname()': (['othercommand_one a b c', 'othercommand_two d e f'], None, 4, False),
+                        'MULTILINE2': (['f1', 'f2'], None, '\t', True),
+                        'MULTILINE': (['e1', 'e2', 'e3'], None, -1, True),
+                        'THIS': (None, None, 0, False),
+                        'OTHER': ('yetanothervalue', None, 0, True)}, newfile3)
+
+
+    def test_edit_metadata_file_4(self):
+
+        newfile4 = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+"""
+
+        self._testeditfile({'NOCHANGE': (None, None, 0, False),
+                        'MULTILINE3': (None, None, 0, False),
+                        'THIS': ('that', None, 0, False),
+                        'do_functionname()': (None, None, 0, False)}, newfile4)
+
+
+    def test_edit_metadata(self):
+        newfile5 = """
+# A comment
+HELLO = "hithere"
+
+# A new comment
+THIS += "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+NEWVAR = "value"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+
+
+        def handle_var(varname, origvalue, op, newlines):
+            if varname == 'THIS':
+                newlines.append('# A new comment\n')
+            elif varname == 'do_functionname()':
+                newlines.append('NEWVAR = "value"\n')
+                newlines.append('\n')
+            valueitem = varvalues.get(varname, None)
+            if valueitem:
+                return valueitem
+            else:
+                return (origvalue, op, 0, True)
+
+        varvalues = {'HELLO': ('hithere', None, 0, True), 'THIS': ('that', '+=', 0, True)}
+        varlist = ['HELLO', 'THIS', 'do_functionname()']
+        (updated, newlines) = bb.utils.edit_metadata(self._origfile.splitlines(True), varlist, handle_var)
+        self.assertTrue(updated, 'List should be updated but isn\'t')
+        self.assertEqual(newlines, newfile5.splitlines(True))