rrs/tools: use layer index lock

We check out different revisions while we do this processing, and so
does the layer index update script, so we shouldn't be allowing both to
run at once or nasty stuff will happen.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

3 files changed, 178 insertions, 153 deletions

diff --git a/rrs/tools/rrs_maintainer_history.py b/rrs/tools/rrs_maintainer_history.py
index 62925fab65..19559ec69d 100755
--- a/rrs/tools/rrs_maintainer_history.py
+++ b/rrs/tools/rrs_maintainer_history.py
@@ -71,6 +71,7 @@ def get_commit_info(info, logger):
     Recreate Maintainership history from the beginning
 """
 def maintainer_history(options, logger):
+    fetchdir = settings.LAYER_FETCH_DIR
     maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
     if not maintplans.exists():
         logger.error('No enabled maintenance plans found')
@@ -78,65 +79,85 @@ def maintainer_history(options, logger):
 
     no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer')
 
-    for maintplan in maintplans:
-        for item in maintplan.maintenanceplanlayerbranch_set.all():
-            layerbranch = item.layerbranch
-            urldir = str(layerbranch.layer.get_fetch_dir())
-            repodir = os.path.join(settings.LAYER_FETCH_DIR, urldir)
-            layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
-
-            utils.runcmd("git checkout master -f", layerdir, logger=logger)
-            maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
-            if not os.path.exists(maintainers_full_path):
-                logger.debug('No maintainers.inc for %s, skipping' % layerbranch)
-                continue
-
-            commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " +
-                    os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger)
-
-            try:
-                with transaction.atomic():
-                    for commit in commits.strip().split("\n"):
-                        if RecipeMaintainerHistory.objects.filter(sha1=commit):
-                            continue
-
-                        logger.debug("Analysing commit %s ..." % (commit))
-
-                        (author_name, author_email, date, title) = \
-                            get_commit_info(utils.runcmd("git show " + commit, repodir,
-                                logger=logger), logger)
-
-                        author = Maintainer.create_or_update(author_name, author_email)
-                        rms = RecipeMaintainerHistory(title=title, date=date, author=author,
-                                sha1=commit)
-                        rms.save()
-
-                        utils.runcmd("git checkout %s -f" % commit,
-                                repodir, logger=logger)
-
-                        lines = [line.strip() for line in open(maintainers_full_path)]
-                        for line in lines:
-                            res = get_recipe_maintainer(line, logger)
-                            if res:
-                                (pn, name, email) = res
-                                qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch)
-
-                                if qry:
-                                    m = Maintainer.create_or_update(name, email)
-
+    lockfn = os.path.join(fetchdir, "layerindex.lock")
+    lockfile = utils.lock_file(lockfn)
+    if not lockfile:
+        logger.error("Layer index lock timeout expired")
+        sys.exit(1)
+    try:
+        for maintplan in maintplans:
+            for item in maintplan.maintenanceplanlayerbranch_set.all():
+                layerbranch = item.layerbranch
+                urldir = str(layerbranch.layer.get_fetch_dir())
+                repodir = os.path.join(fetchdir, urldir)
+                layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+
+                utils.runcmd("git checkout master -f", layerdir, logger=logger)
+                maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
+                if not os.path.exists(maintainers_full_path):
+                    logger.debug('No maintainers.inc for %s, skipping' % layerbranch)
+                    continue
+
+                commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " +
+                        os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger)
+
+                try:
+                    with transaction.atomic():
+                        for commit in commits.strip().split("\n"):
+                            if RecipeMaintainerHistory.objects.filter(sha1=commit):
+                                continue
+
+                            logger.debug("Analysing commit %s ..." % (commit))
+
+                            (author_name, author_email, date, title) = \
+                                get_commit_info(utils.runcmd("git show " + commit, repodir,
+                                    logger=logger), logger)
+
+                            author = Maintainer.create_or_update(author_name, author_email)
+                            rms = RecipeMaintainerHistory(title=title, date=date, author=author,
+                                    sha1=commit)
+                            rms.save()
+
+                            utils.runcmd("git checkout %s -f" % commit,
+                                    repodir, logger=logger)
+
+                            lines = [line.strip() for line in open(maintainers_full_path)]
+                            for line in lines:
+                                res = get_recipe_maintainer(line, logger)
+                                if res:
+                                    (pn, name, email) = res
+                                    qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch)
+
+                                    if qry:
+                                        m = Maintainer.create_or_update(name, email)
+
+                                        rm = RecipeMaintainer()
+                                        rm.recipe = qry[0]
+                                        rm.maintainer = m
+                                        rm.history = rms
+                                        rm.save()
+
+                                        logger.debug("%s: Change maintainer to %s in commit %s." % \
+                                                (pn, m.name, commit))
+                                    else:
+                                        logger.debug("%s: Not found in %s." % \
+                                                (pn, layerbranch))
+
+                            # set missing recipes to no maintainer
+                            for recipe in layerbranch.recipe_set.all():
+                                if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
                                     rm = RecipeMaintainer()
-                                    rm.recipe = qry[0]
-                                    rm.maintainer = m
+                                    rm.recipe = recipe
+                                    rm.maintainer = no_maintainer
                                     rm.history = rms
                                     rm.save()
+                                    logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
+                                                    (recipe.pn, rms.sha1))
 
-                                    logger.debug("%s: Change maintainer to %s in commit %s." % \
-                                            (pn, m.name, commit))
-                                else:
-                                    logger.debug("%s: Not found in %s." % \
-                                            (pn, layerbranch))
+                            utils.runcmd("git checkout master -f", repodir, logger=logger)
 
-                        # set missing recipes to no maintainer
+                        # set new recipes to no maintainer if don't have one
+                        rms = RecipeMaintainerHistory.get_last()
                         for recipe in layerbranch.recipe_set.all():
                             if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
                                 rm = RecipeMaintainer()
@@ -144,26 +165,14 @@ def maintainer_history(options, logger):
                                 rm.maintainer = no_maintainer
                                 rm.history = rms
                                 rm.save()
-                                logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
-                                                (recipe.pn, rms.sha1))
-
-                        utils.runcmd("git checkout master -f", repodir, logger=logger)
-
-                    # set new recipes to no maintainer if don't have one
-                    rms = RecipeMaintainerHistory.get_last()
-                    for recipe in layerbranch.recipe_set.all():
-                        if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
-                            rm = RecipeMaintainer()
-                            rm.recipe = recipe
-                            rm.maintainer = no_maintainer
-                            rm.history = rms
-                            rm.save()
-                            logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
-                                            (recipe.pn))
-                if options.dry_run:
-                    raise DryRunRollbackException
-            except DryRunRollbackException:
-                pass
+                                logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
+                                                (recipe.pn))
+                    if options.dry_run:
+                        raise DryRunRollbackException
+                except DryRunRollbackException:
+                    pass
+    finally:
+        utils.unlock_file(lockfile)
 
 if __name__=="__main__":
     parser = optparse.OptionParser(usage = """%prog [options]""")
diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py
index 1c37bea338..a03b5be2ab 100755
--- a/rrs/tools/rrs_upgrade_history.py
+++ b/rrs/tools/rrs_upgrade_history.py
@@ -93,82 +93,91 @@ def upgrade_history(options, logger):
     if not maintplans.exists():
         logger.error('No enabled maintenance plans found')
         sys.exit(1)
-    for maintplan in maintplans:
-        for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
-            layerbranch = maintplanbranch.layerbranch
-            if options.fullreload and not options.dry_run:
-                RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
-            layer = layerbranch.layer
-            urldir = layer.get_fetch_dir()
-            repodir = os.path.join(fetchdir, urldir)
-            layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
-
-            if maintplanbranch.upgrade_rev and not options.fullreload:
-                initial = False
-                since = maintplanbranch.upgrade_date
-                since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
-            else:
-                initial = True
-                since = options.since
-                since_option = '--since="%s" origin/master' % since
-
-            repo = git.Repo(repodir)
-            assert repo.bare == False
-
-            commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option,
-                                   repodir,
-                                   logger=logger)
-            commit_list = commits.split('\n')
-
-            bitbake_map = {}
-            # Filter out some bad commits
-            bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
-                                bitbakepath,
-                                logger=logger)
-            bitbake_commit_list = bitbake_commits.splitlines()
-            for commit in bitbake_commit_list:
-                bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
-
-            if initial:
-                logger.debug("Adding initial upgrade history ....")
-
-                ct, ctepoch = commit_list.pop(0).split()
-                ctdate = datetime.fromtimestamp(int(ctepoch))
-                run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
-
-            logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d")))
-            for item in commit_list:
-                if item:
-                    ct, ctepoch = item.split()
+
+    lockfn = os.path.join(fetchdir, "layerindex.lock")
+    lockfile = utils.lock_file(lockfn)
+    if not lockfile:
+        logger.error("Layer index lock timeout expired")
+        sys.exit(1)
+    try:
+        for maintplan in maintplans:
+            for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
+                layerbranch = maintplanbranch.layerbranch
+                if options.fullreload and not options.dry_run:
+                    RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
+                layer = layerbranch.layer
+                urldir = layer.get_fetch_dir()
+                repodir = os.path.join(fetchdir, urldir)
+                layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+
+                if maintplanbranch.upgrade_rev and not options.fullreload:
+                    initial = False
+                    since = maintplanbranch.upgrade_date
+                    since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
+                else:
+                    initial = True
+                    since = options.since
+                    since_option = '--since="%s" origin/master' % since
+
+                repo = git.Repo(repodir)
+                assert repo.bare == False
+
+                commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option,
+                                    repodir,
+                                    logger=logger)
+                commit_list = commits.split('\n')
+
+                bitbake_map = {}
+                # Filter out some bad commits
+                bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
+                                    bitbakepath,
+                                    logger=logger)
+                bitbake_commit_list = bitbake_commits.splitlines()
+                for commit in bitbake_commit_list:
+                    bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
+
+                if initial:
+                    logger.debug("Adding initial upgrade history ....")
+
+                    ct, ctepoch = commit_list.pop(0).split()
                     ctdate = datetime.fromtimestamp(int(ctepoch))
-                    commitobj = repo.commit(ct)
-                    touches_recipe = False
-                    for parent in commitobj.parents:
-                        diff = parent.diff(commitobj)
-                        for diffitem in diff:
-                            if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
-                                # We need to look at this commit
-                                touches_recipe = True
+                    run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
+
+                logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d")))
+                for item in commit_list:
+                    if item:
+                        ct, ctepoch = item.split()
+                        ctdate = datetime.fromtimestamp(int(ctepoch))
+                        commitobj = repo.commit(ct)
+                        touches_recipe = False
+                        for parent in commitobj.parents:
+                            diff = parent.diff(commitobj)
+                            for diffitem in diff:
+                                if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
+                                    # We need to look at this commit
+                                    touches_recipe = True
+                                    break
+                            if touches_recipe:
                                 break
-                        if touches_recipe:
-                            break
-                    if not touches_recipe:
-                        # No recipes changed in this commit
-                        # NOTE: Whilst it's possible that a change to a class might alter what's
-                        # in the recipe, we can ignore that since we are only concerned with actual
-                        # upgrades which would always require some sort of change to the recipe
-                        # or an include file, so we can safely skip commits that don't do that
-                        logger.debug("Skipping commit %s" % ct)
-                        continue
-                    logger.debug("Analysing commit %s ..." % ct)
-                    run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
-                    if not options.dry_run:
-                        maintplanbranch.upgrade_rev = ct
-                        maintplanbranch.upgrade_date = ctdate
-                        maintplanbranch.save()
-
-            if commit_list:
-                utils.runcmd("git clean -dfx", repodir, logger=logger)
+                        if not touches_recipe:
+                            # No recipes changed in this commit
+                            # NOTE: Whilst it's possible that a change to a class might alter what's
+                            # in the recipe, we can ignore that since we are only concerned with actual
+                            # upgrades which would always require some sort of change to the recipe
+                            # or an include file, so we can safely skip commits that don't do that
+                            logger.debug("Skipping commit %s" % ct)
+                            continue
+                        logger.debug("Analysing commit %s ..." % ct)
+                        run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
+                        if not options.dry_run:
+                            maintplanbranch.upgrade_rev = ct
+                            maintplanbranch.upgrade_date = ctdate
+                            maintplanbranch.save()
+
+                if commit_list:
+                    utils.runcmd("git clean -dfx", repodir, logger=logger)
+    finally:
+        utils.unlock_file(lockfile)
 
 if __name__=="__main__":
     parser = optparse.OptionParser(usage = """%prog [options]""")
diff --git a/rrs/tools/rrs_upstream_history.py b/rrs/tools/rrs_upstream_history.py
index 5e0590f82c..a3bf7984d0 100755
--- a/rrs/tools/rrs_upstream_history.py
+++ b/rrs/tools/rrs_upstream_history.py
@@ -147,12 +147,17 @@ if __name__=="__main__":
 
     logger.debug("Starting upstream history...")
 
+    maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
+    if not maintplans.exists():
+        logger.error('No enabled maintenance plans found')
+        sys.exit(1)
+
+    lockfn = os.path.join(fetchdir, "layerindex.lock")
+    lockfile = utils.lock_file(lockfn)
+    if not lockfile:
+        logger.error("Layer index lock timeout expired")
+        sys.exit(1)
     try:
-        maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
-        if not maintplans.exists():
-            logger.error('No enabled maintenance plans found')
-            sys.exit(1)
-
         origsyspath = sys.path
 
         for maintplan in maintplans:
@@ -207,3 +212,5 @@ if __name__=="__main__":
                         raise DryRunRollbackException
     except DryRunRollbackException:
         pass
+    finally:
+        utils.unlock_file(lockfile)