data: Fix whitespace on _remove operations

We have some slightly odd behaviours with the current implementation of
_remove operations. For example:

TEST = " A B"
TEST_remove = "C"

would trigger TEST to become "A B" even thought it doesn't contain "C".

In particular, this means that an inactive remove operator added in a
bbappend could change the task checksum which is not desireable.

Fix the operation to preserve whitespace, adding new tests to make this
explict and test further corner cases. Also update the manual to match.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

3 files changed, 22 insertions, 9 deletions

diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
index bc08c814d..fc55ef648 100644
--- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
+++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml
@@ -342,7 +342,7 @@
 
             <para>
                 When you use this syntax, BitBake expects one or more strings.
-                Surrounding spaces are removed as well.
+                Surrounding spaces and spacing are preserved.
                 Here is an example:
                 <literallayout class='monospaced'>
      FOO = "123 456 789 123456 123 456 123 456"
@@ -352,8 +352,8 @@
      FOO2_remove = "abc def"
                 </literallayout>
                 The variable <filename>FOO</filename> becomes
-                "789 123456" and <filename>FOO2</filename> becomes
-                "ghi abcdef".
+                "  789 123456    " and <filename>FOO2</filename> becomes
+                "  ghi abcdef    ".
             </para>
 
             <para>
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 4434142a0..0a8488ca1 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -42,6 +42,7 @@ __setvar_keyword__ = ["_append", "_prepend", "_remove"]
 __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
 __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
+__whitespace_split__ = re.compile('(\s)')
 
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
@@ -818,8 +819,8 @@ class DataSmart(MutableMapping):
 
             if removes:
                 filtered = filter(lambda v: v not in removes,
-                                  value.split())
-                value = " ".join(filtered)
+                                  __whitespace_split__.split(value))
+                value = "".join(filtered)
                 if expand and var in self.expand_cache:
                     # We need to ensure the expand cache has the correct value
                     # flag == "_content" here
diff --git a/lib/bb/tests/data.py b/lib/bb/tests/data.py
index a4a9dd30f..8279115e0 100644
--- a/lib/bb/tests/data.py
+++ b/lib/bb/tests/data.py
@@ -281,7 +281,7 @@ class TestConcatOverride(unittest.TestCase):
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
-        self.assertEqual(self.d.getVar("TEST"), "bar")
+        self.assertEqual(self.d.getVar("TEST"), " bar")
 
     def test_remove_cleared(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
@@ -300,7 +300,7 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
-        self.assertEqual(self.d.getVar("TEST_TEST"), "bar bar")
+        self.assertEqual(self.d.getVar("TEST_TEST"), " bar  bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
@@ -311,13 +311,25 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
         self.d.setVar("TEST_remove", "${BAR}/X")
-        self.assertEqual(self.d.getVar("TEST"), "Y")
+        self.assertEqual(self.d.getVar("TEST"), " Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
         self.d.setVar("TEST_remove", "${BAR}")
-        self.assertEqual(self.d.getVar("TEST"), "A C")
+        self.assertEqual(self.d.getVar("TEST"), "A  C ")
+
+    def test_remove_preserve_whitespace(self):
+        # When the removal isn't active, the original value should be preserved
+        self.d.setVar("TEST", " A B")
+        self.d.setVar("TEST_remove", "C")
+        self.assertEqual(self.d.getVar("TEST"), " A B")
+
+    def test_remove_preserve_whitespace2(self):
+        # When the removal is active preserve the whitespace
+        self.d.setVar("TEST", " A B")
+        self.d.setVar("TEST_remove", "B")
+        self.assertEqual(self.d.getVar("TEST"), " A ")
 
 class TestOverrides(unittest.TestCase):
     def setUp(self):