summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch
blob: f5b99d9447ea0ba9be4d4f3bf2adc72c589c7f37 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
From f34d632c427b1e1570ef9136454fc01d8c9f10a6 Mon Sep 17 00:00:00 2001
From: Robert Yang <liezhi.yang@windriver.com>
Date: Thu, 7 Jul 2016 19:34:09 -0700
Subject: [PATCH] functions.sh: run rpm once to make it faster

The rpm tool is a heavy process, it ran 16 (or 17 for kernel)
"rpm -qp" times when the pkgs are identical, now we only run
"rpm -qp --qf <all we need>" twice (one is for old pkg, and one is for
new), save the results to spec_old and spec_new, then use sed command to
get what we need later, this can make it 75% faster when the pkgs are
identical. Here is the rough data on my host Ubuntu 14.04.4, 32 cores
CPU and 128G mem:
* When the pkgs are identical:
  - Before the patch: 1s
  - After the patch: 0.26s
  I compare the whole spec firstly, and return 0 if they are the same,
  or go on checking one by one if not, without this, it would be 0.46s,
  the gain is great when there are lot of packages, usually, we have
  more than 10,000 rpms to compare.

* When the pkgs are different:
  That depends on where is the different, if the different is at the
  comparing rpmtags stage:
  - Before the patch: 0.26s
  - After the patch: 0.29s
  Increased 0.03s, but if the different is happend later than comparing
  rpmtags, it will save time.

Upstream-Status: Submitted [https://github.com/openSUSE/build-compare/pull/9]

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 functions.sh | 245 ++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 132 insertions(+), 113 deletions(-)
 mode change 100644 => 100755 functions.sh

diff --git a/functions.sh b/functions.sh
old mode 100644
new mode 100755
index b1069d2..aa572f9
--- a/functions.sh
+++ b/functions.sh
@@ -10,9 +10,63 @@
 
 RPM="rpm -qp --nodigest --nosignature"
 
-check_header() 
+# Name, Version, Release
+QF_NAME="%{NAME}"
+QF_VER_REL="%{VERSION}-%{RELEASE}"
+QF_NAME_VER_REL="%{NAME}-%{VERSION}-%{RELEASE}"
+
+# provides destroy this because at least the self-provide includes the
+# -buildnumber :-(
+QF_PROVIDES="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
+QF_PROVIDES="$QF_PROVIDES [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
+QF_PROVIDES="$QF_PROVIDES [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
+QF_PROVIDES="$QF_PROVIDES [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
+
+# don't look at RELEASE, it contains our build number
+QF_TAGS="%{NAME} %{VERSION} %{EPOCH}\\n"
+QF_TAGS="$QF_TAGS %{SUMMARY}\\n%{DESCRIPTION}\\n"
+# the DISTURL tag can be used as checkin ID
+QF_TAGS="$QF_TAGS %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
+QF_TAGS="$QF_TAGS %{LICENSE} %{LICENSE}\\n"
+QF_TAGS="$QF_TAGS %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
+QF_TAGS="$QF_TAGS %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
+QF_TAGS="$QF_TAGS %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
+
+# XXX We also need to check the existence (but not the content (!))
+# of SIGGPG (and perhaps the other SIG*)
+# XXX We don't look at triggers
+QF_TAGS="$QF_TAGS [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
+# Only the first ChangeLog entry; should be enough
+QF_TAGS="$QF_TAGS %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
+
+# scripts, might contain release number
+QF_SCRIPT="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
+
+# Now the files. We leave out mtime and size.  For normal files
+# the size will influence the MD5 anyway.  For directories the sizes can
+# differ, depending on which file system the package was built.  To not
+# have to filter out directories we simply ignore all sizes.
+# Also leave out FILEDEVICES, FILEINODES (depends on the build host),
+# FILECOLORS, FILECLASS (normally useful but file output contains mtimes),
+# FILEDEPENDSX and FILEDEPENDSN.
+# Also FILELANGS (or?)
+QF_FILELIST="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
+# ??? what to do with FILEPROVIDE and FILEREQUIRE?
+
+QF_CHECKSUM="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
+
+QF_ALL="\n___QF_NAME___\n${QF_NAME}\n___QF_NAME___\n"
+QF_ALL="$QF_ALL\n___QF_TAGS___\n${QF_TAGS}\n___QF_TAGS___\n"
+QF_ALL="$QF_ALL\n___QF_VER_REL___\n${QF_VER_REL}\n___QF_VER_REL___\n"
+QF_ALL="$QF_ALL\n___QF_NAME_VER_REL___\n${QF_NAME_VER_REL}\n___QF_NAME_VER_REL___\n"
+QF_ALL="$QF_ALL\n___QF_PROVIDES___\n${QF_PROVIDES}\n___QF_PROVIDES___\n"
+QF_ALL="$QF_ALL\n___QF_SCRIPT___\n${QF_SCRIPT}\n___QF_SCRIPT___\n"
+QF_ALL="$QF_ALL\n___QF_FILELIST___\n${QF_FILELIST}\n___QF_FILELIST___\n"
+QF_ALL="$QF_ALL\n___QF_CHECKSUM___\n${QF_CHECKSUM}\n___QF_CHECKSUM___\n"
+
+check_header()
 {
-   $RPM --qf "$QF" "$1"
+   $RPM --qf "$1" "$2"
 }
 
 # Trim version-release string:
@@ -47,18 +101,6 @@ function grep_release_new()
   grep -E "(/boot|/lib/modules|/lib/firmware|/usr/src)/[^/]+(${version_release_new_regex_l}(\$|[^/]+\$)|${version_release_new_regex_s}(\$|[^/]+\$))"
 }
 
-function check_provides()
-{
-  local pkg=$1
-  # provides destroy this because at least the self-provide includes the
-  # -buildnumber :-(
-  QF="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
-  QF="$QF [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
-  QF="$QF [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
-  QF="$QF [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
-  check_header "$pkg"
-}
-
 #usage unpackage <file> $dir
 # Unpack files in directory $dir
 # like /usr/bin/unpackage - just for one file and with no options
@@ -98,6 +140,30 @@ function unpackage()
     popd 1>/dev/null
 }
 
+# Run diff command on the files
+# $1: printed info
+# $2: file1
+# $3: file2
+function comp_file()
+{
+    echo "comparing $1"
+    if ! diff -au $2 $3; then
+      if test -z "$check_all"; then
+        rm $2 $3 $spec_old $spec_new
+        return 1
+      fi
+    fi
+    return 0
+}
+
+# Get var's value from specfile.
+# $1: var name
+# $2: specfile
+function get_value()
+{
+    sed -n -e "/^___${1}___/,/^___${1}___/p" $2 | sed -e "/^___${1}___/d"
+}
+
 # Compare just the rpm meta data of two rpms
 # Returns:
 # 0 in case of same content
@@ -107,56 +173,29 @@ function unpackage()
 function cmp_spec ()
 {
     local RES
-    local file1 file2
+    local file_old file_new
     local f
     local sh=$1
     local oldrpm=$2
     local newrpm=$3
 
-    QF="%{NAME}"
-    
-    # don't look at RELEASE, it contains our build number
-    QF="$QF %{VERSION} %{EPOCH}\\n"
-    QF="$QF %{SUMMARY}\\n%{DESCRIPTION}\\n"
-    QF="$QF %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
-    QF="$QF %{LICENSE} %{LICENSE}\\n"
-    QF="$QF %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
-    QF="$QF %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
-    QF="$QF %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
-    
- 
-    # XXX We also need to check the existence (but not the content (!))
-    # of SIGGPG (and perhaps the other SIG*)
-    
-    # XXX We don't look at triggers
-    
-    QF="$QF [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
-    
-    # Only the first ChangeLog entry; should be enough
-    QF="$QF %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
-    
-    file1=`mktemp`
-    file2=`mktemp`
-    
-    check_header $oldrpm > $file1
-    check_header $newrpm > $file2
-    
-    # the DISTURL tag can be used as checkin ID
-    #echo "$QF"
-    echo "comparing rpmtags"
-    if ! diff -au $file1 $file2; then
-      if test -z "$check_all"; then
-        rm $file1 $file2
-        return 1
-      fi
-    fi
-    
+    file_old=`mktemp`
+    file_new=`mktemp`
+    spec_old=`mktemp`
+    spec_new=`mktemp`
+
+    check_header "$QF_ALL" $oldrpm > $spec_old
+    check_header "$QF_ALL" $newrpm > $spec_new
+
+    name_new="$(get_value QF_NAME $spec_new)"
+    version_release_new="$(get_value QF_VER_REL $spec_new)"
+    name_ver_rel_new="$(get_value QF_NAME_VER_REL $spec_new)"
+
+    version_release_old="$(get_value QF_VER_REL $spec_old)"
+    name_ver_rel_old="$(get_value QF_NAME_VER_REL $spec_old)"
+
     # Remember to quote the . which is in release
-    version_release_old=$($RPM --qf "%{VERSION}-%{RELEASE}" "$oldrpm")
-    version_release_new=$($RPM --qf "%{VERSION}-%{RELEASE}" "$newrpm")
-    name_ver_rel_old=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$oldrpm")
-    name_ver_rel_new=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$newrpm")
-    # Short version without B_CNT
+    # Short version without B_CN
     version_release_old_regex_s=${version_release_old%.*}
     version_release_old_regex_s=${version_release_old_regex_s//./\\.}
     version_release_new_regex_s=${version_release_new%.*}
@@ -166,10 +205,27 @@ function cmp_spec ()
     version_release_new_regex_l=${version_release_new//./\\.}
     name_ver_rel_old_regex_l=${name_ver_rel_old//./\\.}
     name_ver_rel_new_regex_l=${name_ver_rel_new//./\\.}
+
+    # Check the whole spec file at first, return 0 immediately if the
+    # are the same.
+    cat $spec_old | trim_release_old > $file_old
+    cat $spec_new | trim_release_new > $file_new
+    echo "comparing the whole specfile"
+    if diff -au $spec_old $spec_new; then
+      if test -z "$check_all"; then
+        rm $file_old $file_new $spec_old $spec_new
+        return 0
+      fi
+    fi
+
+    get_value QF_TAGS $spec_old > $file_old
+    get_value QF_TAGS $spec_new > $file_new
+    comp_file rpmtags $file_old $file_new || return 1
+
     # This might happen when?!
     echo "comparing RELEASE"
     if [ "${version_release_old%.*}" != "${version_release_new%.*}" ] ; then
-      case $($RPM --qf '%{NAME}' "$newrpm") in
+      case $name_new in
         kernel-*)
           # Make sure all kernel packages have the same %RELEASE
           echo "release prefix mismatch"
@@ -181,71 +237,34 @@ function cmp_spec ()
         *) ;;
       esac
     fi
-    
-    check_provides $oldrpm | trim_release_old | sort > $file1
-    check_provides $newrpm | trim_release_new | sort > $file2
-    
-    echo "comparing PROVIDES"
-    if ! diff -au $file1 $file2; then
-      if test -z "$check_all"; then
-        rm $file1 $file2
-        return 1
-      fi
-    fi
 
-    # scripts, might contain release number
-    QF="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
-    check_header $oldrpm | trim_release_old > $file1
-    check_header $newrpm | trim_release_new > $file2
+    get_value QF_PROVIDES $spec_old | trim_release_old | sort > $file_old
+    get_value QF_PROVIDES $spec_new | trim_release_new | sort > $file_new
+    comp_file PROVIDES $file_old $file_new || return 1
+
+    get_value QF_SCRIPT $spec_old | trim_release_old > $file_old
+    get_value QF_SCRIPT $spec_new | trim_release_new > $file_new
+    comp_file scripts $file_old $file_new || return 1
 
-    echo "comparing scripts"
-    if ! diff -au $file1 $file2; then
-      if test -z "$check_all"; then
-        rm $file1 $file2
-        return 1
-      fi
-    fi
-    
     # First check the file attributes and later the md5s
-    
-    # Now the files.  We leave out mtime and size.  For normal files
-    # the size will influence the MD5 anyway.  For directories the sizes can
-    # differ, depending on which file system the package was built.  To not
-    # have to filter out directories we simply ignore all sizes.
-    # Also leave out FILEDEVICES, FILEINODES (depends on the build host),
-    # FILECOLORS, FILECLASS (normally useful but file output contains mtimes), 
-    # FILEDEPENDSX and FILEDEPENDSN. 
-    # Also FILELANGS (or?)
-    QF="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
-    # ??? what to do with FILEPROVIDE and FILEREQUIRE?
-
-    check_header $oldrpm | trim_release_old > $file1
-    check_header $newrpm | trim_release_new > $file2
-    
-    echo "comparing filelist"
-    if ! diff -au $file1 $file2; then
-      if test -z "$check_all"; then
-        rm $file1 $file2
-        return 1
-      fi
-    fi
-    
+    get_value QF_FILELIST $spec_old | trim_release_old > $file_old
+    get_value QF_FILELIST $spec_new | trim_release_new > $file_new
+    comp_file filelist $file_old $file_new || return 1
+
     # now the md5sums. if they are different, we check more detailed
     # if there are different filenames, we will already have aborted before
     # file flag 64 means "ghost", filter those out.
-    QF="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
-    check_header $oldrpm |grep -v " 64$"| trim_release_old > $file1
-    check_header $newrpm |grep -v " 64$"| trim_release_new > $file2
-    
+    get_value QF_CHECKSUM $spec_old | grep -v " 64$" | trim_release_old > $file_old
+    get_value QF_CHECKSUM $spec_new | grep -v " 64$" | trim_release_new > $file_new
     RES=2
     # done if the same
     echo "comparing file checksum"
-    if cmp -s $file1 $file2; then
+    if cmp -s $file_old $file_new; then
       RES=0
     fi
-    
+
     # Get only files with different MD5sums
-    files=`diff -U0 $file1 $file2 | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
+    files=`diff -U0 $file_old $file_new | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
 
     if test -f "$sh"; then
       echo "creating rename script"
@@ -261,7 +280,7 @@ function cmp_spec ()
       done >> "${sh}"
     fi
     #
-    rm $file1 $file2
+    rm $file_old $file_new
     return $RES
 }
 
-- 
2.9.0