summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDevendra Tewari <devendra.tewari@gmail.com>2021-04-19 11:23:58 -0300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-04-23 10:12:12 +0100
commit8d48138e9b1f3577cfba64b4757b576c27c6ef04 (patch)
tree9c67de3630ffde438b70e92cf6864103233bc149
parentf2d1523b19cb066a4a06609f036822fe4a8b43f0 (diff)
downloadopenembedded-core-contrib-8d48138e9b1f3577cfba64b4757b576c27c6ef04.tar.gz
classes/lib/scripts: Use shutil.move when os.rename fails
Incremental build in Docker fails with: OSError: [Errno 18] Invalid cross-device link when source and destination are on different overlay filesystems. This change handles error with os.rename and retries with shutil.move. The reason os.rename is still used is because shutil.move is too slow for speed sensitive sections of code. [YOCTO #14301] Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildhistory.bbclass6
-rw-r--r--meta/classes/package.bbclass12
-rw-r--r--meta/classes/populate_sdk_ext.bbclass10
-rw-r--r--meta/classes/reproducible_build.bbclass6
-rw-r--r--meta/classes/sstate.bbclass22
-rw-r--r--meta/classes/update-alternatives.bbclass11
-rw-r--r--meta/lib/oe/package_manager/deb/__init__.py18
-rw-r--r--meta/lib/oe/package_manager/ipk/__init__.py5
-rw-r--r--meta/lib/oe/rootfs.py15
-rw-r--r--meta/lib/oeqa/selftest/cases/wic.py16
-rwxr-xr-xscripts/combo-layer5
-rw-r--r--scripts/lib/devtool/standard.py21
-rw-r--r--scripts/lib/devtool/upgrade.py7
-rw-r--r--scripts/lib/wic/plugins/imager/direct.py5
14 files changed, 130 insertions, 29 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 49af61c9c5..554c8810ac 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -875,7 +875,11 @@ python buildhistory_eventhandler() {
entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
bb.utils.mkdirhier(olddir)
for entry in entries:
- os.rename(os.path.join(rootdir, entry),
+ try:
+ os.rename(os.path.join(rootdir, entry),
+ os.path.join(olddir, entry))
+ except OSError:
+ shutil.move(os.path.join(rootdir, entry),
os.path.join(olddir, entry))
elif isinstance(e, bb.event.BuildCompleted):
if reset:
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index e3f0a7060b..6707c2e9db 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -835,6 +835,7 @@ perform_packagecopy[dirs] = "${PKGD}"
# the fs-perms.txt files
python fixup_perms () {
import pwd, grp
+ import shutil
# init using a string with the same format as a line as documented in
# the fs-perms.txt file
@@ -1049,7 +1050,10 @@ python fixup_perms () {
# Create path to move directory to, move it, and then setup the symlink
bb.utils.mkdirhier(os.path.dirname(target))
#bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget))
- os.rename(origin, target)
+ try:
+ os.rename(origin, target)
+ except OSError:
+ shutil.move(origin, target)
#bb.note("Fixup Perms: Link %s -> %s" % (dir, link))
os.symlink(link, origin)
@@ -1776,6 +1780,7 @@ python package_do_shlibs() {
import itertools
import re, pipes
import subprocess
+ import shutil
exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False)
if exclude_shlibs:
@@ -1967,7 +1972,10 @@ python package_do_shlibs() {
for (old, new) in renames:
bb.note("Renaming %s to %s" % (old, new))
- os.rename(old, new)
+ try:
+ os.rename(old, new)
+ except OSError:
+ shutil.move(old, new)
pkgfiles[pkg].remove(old)
shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 84232ed9f5..12e758c2ad 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -165,7 +165,10 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
shutil.rmtree(temp_sdkbasepath)
except FileNotFoundError:
pass
- os.rename(sdkbasepath, temp_sdkbasepath)
+ try:
+ os.rename(sdkbasepath, temp_sdkbasepath)
+ except OSError:
+ shutil.move(sdkbasepath, temp_sdkbasepath)
cmdprefix = '. %s .; ' % conf_initpath
logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt'
try:
@@ -175,7 +178,10 @@ def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
bb.fatal(msg)
- os.rename(temp_sdkbasepath, sdkbasepath)
+ try:
+ os.rename(temp_sdkbasepath, sdkbasepath)
+ except OSError:
+ shutil.move(temp_sdkbasepath, sdkbasepath)
# Clean out residue of running bitbake, which check_sstate_task_list()
# will effectively do
clean_esdk_builddir(d, sdkbasepath)
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass
index f06e00d70d..480d58f86d 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -57,13 +57,17 @@ do_deploy_source_date_epoch () {
}
python do_deploy_source_date_epoch_setscene () {
+ import shutil
sstate_setscene(d)
bb.utils.mkdirhier(d.getVar('SDE_DIR'))
sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
if os.path.exists(sde_file):
target = d.getVar('SDE_FILE')
bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
- os.rename(sde_file, target)
+ try:
+ os.rename(sde_file, target)
+ except OSError:
+ shutil.move(sde_file, target)
else:
bb.debug(1, "%s not found!" % sde_file)
}
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 8e8efd18d5..60f7a94285 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):
def sstate_installpkgdir(ss, d):
import oe.path
import subprocess
+ import shutil
sstateinst = d.getVar("SSTATE_INSTDIR")
d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
@@ -401,7 +402,10 @@ def sstate_installpkgdir(ss, d):
for state in ss['dirs']:
prepdir(state[1])
- os.rename(sstateinst + state[0], state[1])
+ try:
+ os.rename(sstateinst + state[0], state[1])
+ except OSError:
+ shutil.move(sstateinst + state[0], state[1])
sstate_install(ss, d)
for plain in ss['plaindirs']:
@@ -413,7 +417,10 @@ def sstate_installpkgdir(ss, d):
dest = plain
bb.utils.mkdirhier(src)
prepdir(dest)
- os.rename(src, dest)
+ try:
+ os.rename(src, dest)
+ except OSError:
+ shutil.move(src, dest)
return True
@@ -638,6 +645,7 @@ python sstate_hardcode_path () {
def sstate_package(ss, d):
import oe.path
+ import shutil
tmpdir = d.getVar('TMPDIR')
@@ -664,7 +672,10 @@ def sstate_package(ss, d):
continue
bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
- os.rename(state[1], sstatebuild + state[0])
+ try:
+ os.rename(state[1], sstatebuild + state[0])
+ except OSError:
+ shutil.move(state[1], sstatebuild + state[0])
workdir = d.getVar('WORKDIR')
sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
@@ -674,7 +685,10 @@ def sstate_package(ss, d):
pdir = plain.replace(sharedworkdir, sstatebuild)
bb.utils.mkdirhier(plain)
bb.utils.mkdirhier(pdir)
- os.rename(plain, pdir)
+ try:
+ os.rename(plain, pdir)
+ except OSError:
+ shutil.move(plain, pdir)
d.setVar('SSTATE_BUILDDIR', sstatebuild)
d.setVar('SSTATE_INSTDIR', sstatebuild)
diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index 8c2b66e7f1..7e213472e3 100644
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -139,6 +139,7 @@ python apply_update_alternative_renames () {
return
import re
+ import shutil
def update_files(alt_target, alt_target_rename, pkg, d):
f = d.getVar('FILES_' + pkg)
@@ -184,7 +185,10 @@ python apply_update_alternative_renames () {
link_rename.append((alt_target, alt_target_rename))
else:
bb.note('%s: Rename %s -> %s' % (pn, alt_target, alt_target_rename))
- os.rename(src, dest)
+ try:
+ os.rename(src, dest)
+ except OSError:
+ shutil.move(src, dest)
update_files(alt_target, alt_target_rename, pkg, d)
else:
bb.warn("%s: alternative target (%s or %s) does not exist, skipping..." % (pn, alt_target, alt_target_rename))
@@ -201,7 +205,10 @@ python apply_update_alternative_renames () {
if os.path.lexists(link_target):
# Ok, the link_target exists, we can rename
bb.note('%s: Rename (link) %s -> %s' % (pn, alt_target, alt_target_rename))
- os.rename(src, dest)
+ try:
+ os.rename(src, dest)
+ except OSError:
+ shutil.move(src, dest)
else:
# Try to resolve the broken link to link.${BPN}
link_maybe = '%s.%s' % (os.readlink(src), pn)
diff --git a/meta/lib/oe/package_manager/deb/__init__.py b/meta/lib/oe/package_manager/deb/__init__.py
index 2ee68fefb1..8edc653dff 100644
--- a/meta/lib/oe/package_manager/deb/__init__.py
+++ b/meta/lib/oe/package_manager/deb/__init__.py
@@ -5,6 +5,7 @@
import re
import subprocess
from oe.package_manager import *
+import shutil
class DpkgIndexer(Indexer):
def _create_configs(self):
@@ -214,7 +215,10 @@ class DpkgPM(OpkgDpkgPM):
tmp_sf.write(status)
- os.rename(status_file + ".tmp", status_file)
+ try:
+ os.rename(status_file + ".tmp", status_file)
+ except OSError:
+ shutil.move(status_file + ".tmp", status_file)
def run_pre_post_installs(self, package_name=None):
"""
@@ -299,13 +303,21 @@ class DpkgPM(OpkgDpkgPM):
for dir in dirs:
new_dir = re.sub(r"\.dpkg-new", "", dir)
if dir != new_dir:
- os.rename(os.path.join(root, dir),
+ try:
+ os.rename(os.path.join(root, dir),
+ os.path.join(root, new_dir))
+ except OSError:
+ shutil.move(os.path.join(root, dir),
os.path.join(root, new_dir))
for file in files:
new_file = re.sub(r"\.dpkg-new", "", file)
if file != new_file:
- os.rename(os.path.join(root, file),
+ try:
+ os.rename(os.path.join(root, file),
+ os.path.join(root, new_file))
+ except OSError:
+ shutil.move(os.path.join(root, file),
os.path.join(root, new_file))
diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index da488c1c7f..2aa21949f3 100644
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -213,7 +213,10 @@ class OpkgPM(OpkgDpkgPM):
tmp_sf.write(status)
- os.rename(status_file + ".tmp", status_file)
+ try:
+ os.rename(status_file + ".tmp", status_file)
+ except OSError:
+ shutil.move(status_file + ".tmp", status_file)
def _create_custom_config(self):
bb.note("Building from feeds activated!")
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 5f81023040..ef00010047 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -114,7 +114,10 @@ class Rootfs(object, metaclass=ABCMeta):
shutil.rmtree(self.image_rootfs + '-orig')
except:
pass
- os.rename(self.image_rootfs, self.image_rootfs + '-orig')
+ try:
+ os.rename(self.image_rootfs, self.image_rootfs + '-orig')
+ except OSError:
+ shutil.move(self.image_rootfs, self.image_rootfs + '-orig')
bb.note(" Creating debug rootfs...")
bb.utils.mkdirhier(self.image_rootfs)
@@ -165,10 +168,16 @@ class Rootfs(object, metaclass=ABCMeta):
shutil.rmtree(self.image_rootfs + '-dbg')
except:
pass
- os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
+ try:
+ os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
+ except OSError:
+ shutil.move(self.image_rootfs, self.image_rootfs + '-dbg')
bb.note(" Restoreing original rootfs...")
- os.rename(self.image_rootfs + '-orig', self.image_rootfs)
+ try:
+ os.rename(self.image_rootfs + '-orig', self.image_rootfs)
+ except OSError:
+ shutil.move(self.image_rootfs + '-orig', self.image_rootfs)
def _exec_shell_cmd(self, cmd):
fakerootcmd = self.d.getVar('FAKEROOT')
diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index fa81584a8c..fa93973087 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -1269,6 +1269,7 @@ class Wic2(WicTestCase):
def test_expand_mbr_image(self):
"""Test wic write --expand command for mbr image"""
+ import shutil
# build an image
config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "directdisk.wks"\n'
self.append_config(config)
@@ -1306,8 +1307,14 @@ class Wic2(WicTestCase):
result = runCmd("%s/usr/sbin/sfdisk -F %s" % (sysroot, new_image_path))
self.assertTrue("0 B, 0 bytes, 0 sectors" in result.output)
- os.rename(image_path, image_path + '.bak')
- os.rename(new_image_path, image_path)
+ try:
+ os.rename(image_path, image_path + '.bak')
+ except OSError:
+ shutil.move(image_path, image_path + '.bak')
+ try:
+ os.rename(new_image_path, image_path)
+ except OSError:
+ shutil.move(new_image_path, image_path)
# Check if it boots in qemu
with runqemu('core-image-minimal', ssh=False) as qemu:
@@ -1318,7 +1325,10 @@ class Wic2(WicTestCase):
if os.path.exists(new_image_path):
os.unlink(new_image_path)
if os.path.exists(image_path + '.bak'):
- os.rename(image_path + '.bak', image_path)
+ try:
+ os.rename(image_path + '.bak', image_path)
+ except OSError:
+ shutil.move(image_path + '.bak', image_path)
def test_wic_ls_ext(self):
"""Test listing content of the ext partition using 'wic ls'"""
diff --git a/scripts/combo-layer b/scripts/combo-layer
index a634dd69d2..1b88340b6e 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -508,7 +508,10 @@ def check_patch(patchfile):
f.close()
if of:
of.close()
- os.rename(patchfile + '.tmp', patchfile)
+ try:
+ os.rename(patchfile + '.tmp', patchfile)
+ except OSError:
+ shutil.move(patchfile + '.tmp', patchfile)
def drop_to_shell(workdir=None):
if not sys.stdin.isatty():
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index f364a45283..3854dfb3e8 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -746,7 +746,10 @@ def _check_preserve(config, recipename):
os.remove(removefile)
else:
tf.write(line)
- os.rename(newfile, origfile)
+ try:
+ os.rename(newfile, origfile)
+ except OSError:
+ shutil.move(newfile, origfile)
def get_staging_kver(srcdir):
# Kernel version from work-shared
@@ -1094,10 +1097,16 @@ def rename(args, config, basepath, workspace):
# Rename bbappend
logger.info('Renaming %s to %s' % (append, newappend))
- os.rename(append, newappend)
+ try:
+ os.rename(append, newappend)
+ except OSError:
+ shutil.move(append, newappend)
# Rename recipe file
logger.info('Renaming %s to %s' % (recipefile, newfile))
- os.rename(recipefile, newfile)
+ try:
+ os.rename(recipefile, newfile)
+ except OSError:
+ shutil.move(recipefile, newfile)
# Rename source tree if it's the default path
appendmd5 = None
@@ -1333,7 +1342,11 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
if match_name:
# Rename patch files
if new_patch != match_name:
- os.rename(os.path.join(destdir, new_patch),
+ try:
+ os.rename(os.path.join(destdir, new_patch),
+ os.path.join(destdir, match_name))
+ except OSError:
+ shutil.move(os.path.join(destdir, new_patch),
os.path.join(destdir, match_name))
# Need to pop it off the list now before checking changed_revs
oldpath = existing_patches.pop(old_patch)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 5a057e95f5..2bff69f5a2 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -71,7 +71,12 @@ def _rename_recipe_dirs(oldpv, newpv, path):
if oldfile.find(oldpv) != -1:
newfile = oldfile.replace(oldpv, newpv)
if oldfile != newfile:
- os.rename(os.path.join(path, oldfile), os.path.join(path, newfile))
+ try:
+ os.rename(os.path.join(path, oldfile),
+ os.path.join(path, newfile))
+ except OSError:
+ shutil.move(os.path.join(path, oldfile),
+ os.path.join(path, newfile))
def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
oldrecipe = os.path.basename(oldrecipe)
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index ea709e8c54..fb5e52b2a6 100644
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -616,5 +616,8 @@ class PartitionedImage():
part.start + part.size_sec - 1, part.size_sec)
partimage = self.path + '.p%d' % part.num
- os.rename(source, partimage)
+ try:
+ os.rename(source, partimage)
+ except OSError:
+ shutil.move(source, partimage)
self.partimages.append(partimage)