rpm: Handle proper return value to avoid major issues

0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch changed
to avoid critical issues
Handled return values of getrlimit() and lzma_cputhreads() functions
to avoid unexpected behaviours like devide by zero and potential read
of uninitialized variable 'virtual_memory'
Upstream-Status: Pending [merge of multithreading patches to upstream]

Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 5aae9c2cb464350bc443a0f60fd6602942e61f46)
Signed-off-by: Steve Sakoman <steve@sakoman.com>

1 files changed, 14 insertions, 11 deletions

diff --git a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
index 6454785254..dc3f74fecd 100644
--- a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
+++ b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
@@ -11,36 +11,39 @@ CPU thread.
 Upstream-Status: Pending [merge of multithreading patches to upstream]
 
 Signed-off-by: Peter Bergin <peter@berginkonsult.se>
+Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
 ---
- rpmio/rpmio.c | 34 ++++++++++++++++++++++++++++++++++
- 1 file changed, 34 insertions(+)
+ rpmio/rpmio.c | 36 ++++++++++++++++++++++++++++++++++++
+ 1 file changed, 36 insertions(+)
 
 diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
 index e051c98..b3c56b6 100644
 --- a/rpmio/rpmio.c
 +++ b/rpmio/rpmio.c
-@@ -845,6 +845,40 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
+@@ -845,6 +845,42 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
  		}
  #endif
  
-+		struct rlimit virtual_memory;
-+		getrlimit(RLIMIT_AS, &virtual_memory);
-+		if (virtual_memory.rlim_cur != RLIM_INFINITY) {
++		struct rlimit virtual_memory = {RLIM_INFINITY , RLIM_INFINITY};
++		int status = getrlimit(RLIMIT_AS, &virtual_memory);
++		if ((status != -1) && (virtual_memory.rlim_cur != RLIM_INFINITY)) {
 +			const uint64_t virtual_memlimit = virtual_memory.rlim_cur;
++			uint32_t threads_max = lzma_cputhreads();
 +			const uint64_t virtual_memlimit_per_cpu_thread =
-+				virtual_memlimit / lzma_cputhreads();
-+			uint64_t memory_usage_virt;
++				virtual_memlimit / ((threads_max == 0) ? 1 : threads_max);
 +			rpmlog(RPMLOG_NOTICE, "XZ: virtual memory restricted to %lu and "
 +			       "per CPU thread %lu\n", virtual_memlimit, virtual_memlimit_per_cpu_thread);
++			uint64_t memory_usage_virt;
 +			/* keep reducing the number of compression threads until memory
 +			   usage falls below the limit per CPU thread*/
 +			while ((memory_usage_virt = lzma_stream_encoder_mt_memusage(&mt_options)) >
 +			       virtual_memlimit_per_cpu_thread) {
-+				/* If number of threads goes down to zero lzma_stream_encoder will
-+				 * will return UINT64_MAX. We must check here to avoid an infinite loop.
++				/* If number of threads goes down to zero or in case of any other error
++				 * lzma_stream_encoder_mt_memusage will return UINT64_MAX. We must check
++				 * for both the cases here to avoid an infinite loop.
 +				 * If we get into situation that one thread requires more virtual memory
 +				 * than available we set one thread, print error message and try anyway. */
-+				if (--mt_options.threads == 0) {
++				if ((--mt_options.threads == 0) || (memory_usage_virt == UINT64_MAX)) {
 +					mt_options.threads = 1;
 +					rpmlog(RPMLOG_WARNING,
 +					       "XZ: Could not adjust number of threads to get below "