From fac06aaf2a2b79adc2fbfc7fbd5e6526ff2b515c Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 20 Aug 2010 11:24:02 +0100 Subject: bitbake/persist_data: Attempt to fix locking issues It appears the timeout sometimes has no effect and we see database access failures. Combat this by wrapping the execute function in all cases and retrying manually ourselves. Thanks to Kevin Tian for help debugging this. Signed-off-by: Richard Purdie --- bitbake/lib/bb/persist_data.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py index df0409cd8a..00f4929945 100644 --- a/bitbake/lib/bb/persist_data.py +++ b/bitbake/lib/bb/persist_data.py @@ -67,20 +67,20 @@ class PersistData: Should be called before any domain is used Creates it if it doesn't exist. """ - self.cursor.execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain) + self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" % domain) def delDomain(self, domain): """ Removes a domain and all the data it contains """ - self.cursor.execute("DROP TABLE IF EXISTS %s;" % domain) + self._execute("DROP TABLE IF EXISTS %s;" % domain) def getKeyValues(self, domain): """ Return a list of key + value pairs for a domain """ ret = {} - data = self.cursor.execute("SELECT key, value from %s;" % domain) + data = self._execute("SELECT key, value from %s;" % domain) for row in data: ret[str(row[0])] = str(row[1]) @@ -90,7 +90,7 @@ class PersistData: """ Return the value of a key for a domain """ - data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key]) + data = self._execute("SELECT * from %s where key=?;" % domain, [key]) for row in data: return row[1] @@ -98,7 +98,7 @@ class PersistData: """ Sets the value of a key for a domain """ - data = self.cursor.execute("SELECT * from %s where key=?;" % domain, [key]) + data = self._execute("SELECT * from %s where key=?;" % domain, [key]) rows = 0 for row in data: rows = rows + 1 @@ -113,12 +113,21 @@ class PersistData: """ self._execute("DELETE from %s where key=?;" % domain, [key]) + # + # We wrap the sqlite execute calls as on contended machines or single threaded + # systems we can have multiple processes trying to access the DB at once and it seems + # sqlite sometimes doesn't wait for the timeout. We therefore loop but put in an + # emergency brake too + # def _execute(self, *query): + count = 0 while True: try: - self.cursor.execute(*query) - return + ret = self.cursor.execute(*query) + #print "Had to retry %s times" % count + return ret except sqlite3.OperationalError as e: - if 'database is locked' in str(e): + if 'database is locked' in str(e) and count < 500: + count = count + 1 continue raise -- cgit 1.2.3-korg