diff options
author | Devendra Tewari <devendra.tewari@gmail.com> | 2021-04-19 11:23:58 -0300 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-05-06 11:04:34 +0100 |
commit | c5c4e49574ab2a65e06298a0a77bb98b041cf56b (patch) | |
tree | 0e7a867e7cd9f2dd814aef10fd1a65d68c99cf20 | |
parent | 2db571324f755edc4981deecbcfdf0aaa5a97627 (diff) | |
download | bitbake-contrib-c5c4e49574ab2a65e06298a0a77bb98b041cf56b.tar.gz |
lib/bb: Add bb.utils.rename() helper function and use for renaming
os.rename can fail for example an incremental build in Docker fails with:
OSError: [Errno 18] Invalid cross-device link
when source and destination are on different overlay filesystems.
Rather than trying to fix every call site, add a wrapper in bb.utils
for renames. We can then handle cross device failures and
fall back to shutil.move. The reason os.rename is still used is
because shutil.move is too slow for speed sensitive sections of code.
[YOCTO #14301]
Signed-off-by: Devendra Tewari <devendra.tewari@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | lib/bb/siggen.py | 2 | ||||
-rw-r--r-- | lib/bb/tests/fetch.py | 4 | ||||
-rw-r--r-- | lib/bb/utils.py | 18 |
3 files changed, 18 insertions, 6 deletions
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py index f3fa3000f..07692e673 100644 --- a/lib/bb/siggen.py +++ b/lib/bb/siggen.py @@ -402,7 +402,7 @@ class SignatureGeneratorBasic(SignatureGenerator): p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) - os.rename(tmpfile, sigfile) + bb.utils.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 0f7585e11..9291ce4a0 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1798,7 +1798,7 @@ class GitShallowTest(FetcherTest): # Set up the mirror mirrordir = os.path.join(self.tempdir, 'mirror') - os.rename(self.dldir, mirrordir) + bb.utils.rename(self.dldir, mirrordir) self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) # Fetch from the mirror @@ -1916,7 +1916,7 @@ class GitShallowTest(FetcherTest): bb.utils.mkdirhier(mirrordir) self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir) - os.rename(os.path.join(self.dldir, mirrortarball), + bb.utils.rename(os.path.join(self.dldir, mirrortarball), os.path.join(mirrordir, mirrortarball)) # Fetch from the mirror diff --git a/lib/bb/utils.py b/lib/bb/utils.py index b282d09ab..6ba1d2a37 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -782,7 +782,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]: try: - os.rename(src, destpath) + bb.utils.rename(src, destpath) renamefailed = 0 except Exception as e: if e.errno != errno.EXDEV: @@ -796,7 +796,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if stat.S_ISREG(sstat[stat.ST_MODE]): try: # For safety copy then move it over. shutil.copyfile(src, destpath + "#new") - os.rename(destpath + "#new", destpath) + bb.utils.rename(destpath + "#new", destpath) didcopy = 1 except Exception as e: print('movefile: copy', src, '->', dest, 'failed.', e) @@ -874,7 +874,7 @@ def copyfile(src, dest, newmtime = None, sstat = None): # For safety copy then move it over. shutil.copyfile(src, dest + "#new") - os.rename(dest + "#new", dest) + bb.utils.rename(dest + "#new", dest) except Exception as e: logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e)) return False @@ -1669,3 +1669,15 @@ def is_semver(version): return False return True + +# Wrapper around os.rename which can handle cross device problems +# e.g. from container filesystems +def rename(src, dst): + try: + os.rename(src, dst) + except OSError as err: + if err.errno == 18: + # Invalid cross-device link error + shutil.move(src, dst) + else: + raise err |