Fix import script for Django 1.8 & Python 3

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

1 files changed, 163 insertions, 166 deletions

diff --git a/layerindex/tools/import_layer.py b/layerindex/tools/import_layer.py
index b472d2aea9..0521e1ce75 100755
--- a/layerindex/tools/import_layer.py
+++ b/layerindex/tools/import_layer.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Import a layer into the database
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2016 Intel Corporation
 # Author: Paul Eggleton <paul.eggleton@linux.intel.com>
 #
 # Licensed under the MIT license, see COPYING.MIT for details
@@ -20,6 +20,9 @@ import utils
 import logging
 import subprocess
 
+class DryRunRollbackException(Exception):
+    pass
+
 logger = utils.logger_create('LayerIndexImport')
 
 link_re = re.compile(r'\[(http.*) +link\]')
@@ -138,12 +141,12 @@ def maintainers_extract(maintfn):
 
 
 def get_github_layerinfo(layer_url, username = None, password = None):
-    import httplib
+    import http.client
     import json
     from layerindex.models import LayerMaintainer
 
     def github_api_call(path):
-        conn = httplib.HTTPSConnection('api.github.com')
+        conn = http.client.HTTPSConnection('api.github.com')
         headers = {"User-Agent": "test_github.py"}
         if username:
             import base64
@@ -161,18 +164,18 @@ def get_github_layerinfo(layer_url, username = None, password = None):
         layer_url = layer_url[:-4]
     resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
     if resp.status in [200, 302]:
-        data = resp.read()
+        data = resp.read().decode('utf-8')
         json_data = json.loads(data)
         #headers = dict((key, value) for key, value in resp.getheaders())
         #print(headers)
         owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
         if resp.status in [200, 302]:
-            owner_data = owner_resp.read()
+            owner_data = owner_resp.read().decode('utf-8')
             owner_json_data = json.loads(owner_data)
         else:
-            logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read()))
+            logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
     else:
-        logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read()))
+        logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
 
     return (json_data, owner_json_data)
 
@@ -212,7 +215,7 @@ def main():
         if options.subdir:
             layer_name = options.subdir
         else:
-            layer_name = filter(None, layer_url.split('/'))[-1]
+            layer_name = [x for x in layer_url.split('/') if x][-1]
             if layer_name.endswith('.git'):
                 layer_name = layer_name[:-4]
 
@@ -244,168 +247,162 @@ def main():
 
     master_branch = utils.get_branch('master')
     core_layer = None
-    transaction.enter_transaction_management()
-    transaction.managed(True)
     try:
-        # Fetch layer
-        logger.info('Fetching repository %s' % layer_url)
-
-        layer = LayerItem()
-        layer.name = layer_name
-        layer.status = 'P'
-        layer.layer_type = 'M'
-        layer.summary = 'tempvalue'
-        layer.description = layer.summary
-
-        set_vcs_fields(layer, layer_url)
-
-        urldir = layer.get_fetch_dir()
-        repodir = os.path.join(fetchdir, urldir)
-        out = None
-        try:
-            if not os.path.exists(repodir):
-                out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
-            else:
-                out = utils.runcmd("git fetch", repodir, logger=logger)
-        except Exception as e:
-            logger.error("Fetch failed: %s" % str(e))
-            sys.exit(1)
-
-        actual_branch = ''
-        try:
-            out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
-        except subprocess.CalledProcessError:
-            branches = utils.runcmd("git branch -r", repodir, logger=logger)
-            for line in branches.splitlines():
-                if 'origin/HEAD ->' in line:
-                    actual_branch = line.split('-> origin/')[-1]
-                    break
-            if not actual_branch:
-                logger.error("Repository has no master branch nor origin/HEAD")
-                sys.exit(1)
-            out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
-
-        layer_paths = []
-        if options.subdir:
-            layerdir = os.path.join(repodir, options.subdir)
-            if not os.path.exists(layerdir):
-                logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
-                sys.exit(1)
-            if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
-                logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
-                sys.exit(1)
-            layer_paths.append(layerdir)
-        else:
-            if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
-                layer_paths.append(repodir)
-            # Find subdirs with a conf/layer.conf
-            for subdir in os.listdir(repodir):
-                subdir_path = os.path.join(repodir, subdir)
-                if os.path.isdir(subdir_path):
-                    if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
-                        layer_paths.append(subdir_path)
-            if not layer_paths:
-                logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
+        with transaction.atomic():
+            # Fetch layer
+            logger.info('Fetching repository %s' % layer_url)
+
+            layer = LayerItem()
+            layer.name = layer_name
+            layer.status = 'P'
+            layer.layer_type = 'M'
+            layer.summary = 'tempvalue'
+            layer.description = layer.summary
+
+            set_vcs_fields(layer, layer_url)
+
+            urldir = layer.get_fetch_dir()
+            repodir = os.path.join(fetchdir, urldir)
+            out = None
+            try:
+                if not os.path.exists(repodir):
+                    out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
+                else:
+                    out = utils.runcmd("git fetch", repodir, logger=logger)
+            except Exception as e:
+                logger.error("Fetch failed: %s" % str(e))
                 sys.exit(1)
 
-        if 'github.com' in layer.vcs_url:
-            json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
-
-        for layerdir in layer_paths:
-            layer.pk = None
-            if layerdir != repodir:
-                subdir = os.path.relpath(layerdir, repodir)
-                if len(layer_paths) > 1:
-                    layer.name = subdir
+            actual_branch = ''
+            try:
+                out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
+            except subprocess.CalledProcessError:
+                branches = utils.runcmd("git branch -r", repodir, logger=logger)
+                for line in branches.splitlines():
+                    if 'origin/HEAD ->' in line:
+                        actual_branch = line.split('-> origin/')[-1]
+                        break
+                if not actual_branch:
+                    logger.error("Repository has no master branch nor origin/HEAD")
+                    sys.exit(1)
+                out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
+
+            layer_paths = []
+            if options.subdir:
+                layerdir = os.path.join(repodir, options.subdir)
+                if not os.path.exists(layerdir):
+                    logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
+                    sys.exit(1)
+                if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+                    logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
+                    sys.exit(1)
+                layer_paths.append(layerdir)
             else:
-                subdir = ''
-            if LayerItem.objects.filter(name=layer.name).exists():
-                logger.error('A layer named "%s" already exists in the database' % layer_name)
-                sys.exit(1)
-
-            logger.info('Creating layer %s' % layer.name)
-            # Guess layer type
-            if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
-                layer.layer_type = 'D'
-            elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
-                layer.layer_type = 'B'
-            layer.save()
-            layerbranch = LayerBranch()
-            layerbranch.layer = layer
-            layerbranch.branch = master_branch
-            if layerdir != repodir:
-                layerbranch.vcs_subdir = subdir
-            if actual_branch:
-                layerbranch.actual_branch = actual_branch
-            layerbranch.save()
-            if layer.name != settings.CORE_LAYER_NAME:
-                if not core_layer:
-                    core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
-                if core_layer:
-                    layerdep = LayerDependency()
-                    layerdep.layerbranch = layerbranch
-                    layerdep.dependency = core_layer
-                    layerdep.save()
-
-            # Get some extra meta-information
-            readme_files = glob.glob(os.path.join(layerdir, 'README*'))
-            if (not readme_files) and subdir:
-                readme_files = glob.glob(os.path.join(repodir, 'README*'))
-            maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
-            if (not maintainer_files) and subdir:
-                maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
-
-            maintainers = []
-            if readme_files:
-                (desc, maintainers, deps) = readme_extract(readme_files[0])
-                if desc:
-                    layer.summary = layer.name
-                    layer.description = desc
-            if maintainer_files:
-                maintainers.extend(maintainers_extract(readme_files[0]))
-
-            if (not maintainers) and 'github.com' in layer.vcs_url:
-                if json_data:
-                    layer.summary = json_data['description']
+                if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
+                    layer_paths.append(repodir)
+                # Find subdirs with a conf/layer.conf
+                for subdir in os.listdir(repodir):
+                    subdir_path = os.path.join(repodir, subdir)
+                    if os.path.isdir(subdir_path):
+                        if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
+                            layer_paths.append(subdir_path)
+                if not layer_paths:
+                    logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
+                    sys.exit(1)
+
+            if 'github.com' in layer.vcs_url:
+                json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
+
+            for layerdir in layer_paths:
+                layer.pk = None
+                if layerdir != repodir:
+                    subdir = os.path.relpath(layerdir, repodir)
+                    if len(layer_paths) > 1:
+                        layer.name = subdir
+                else:
+                    subdir = ''
+                if LayerItem.objects.filter(name=layer.name).exists():
+                    logger.error('A layer named "%s" already exists in the database' % layer_name)
+                    sys.exit(1)
+
+                logger.info('Creating layer %s' % layer.name)
+                # Guess layer type
+                if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
+                    layer.layer_type = 'D'
+                elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
+                    layer.layer_type = 'B'
+                layer.save()
+                layerbranch = LayerBranch()
+                layerbranch.layer = layer
+                layerbranch.branch = master_branch
+                if layerdir != repodir:
+                    layerbranch.vcs_subdir = subdir
+                if actual_branch:
+                    layerbranch.actual_branch = actual_branch
+                layerbranch.save()
+                if layer.name != settings.CORE_LAYER_NAME:
+                    if not core_layer:
+                        core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
+                    if core_layer:
+                        layerdep = LayerDependency()
+                        layerdep.layerbranch = layerbranch
+                        layerdep.dependency = core_layer
+                        layerdep.save()
+
+                # Get some extra meta-information
+                readme_files = glob.glob(os.path.join(layerdir, 'README*'))
+                if (not readme_files) and subdir:
+                    readme_files = glob.glob(os.path.join(repodir, 'README*'))
+                maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
+                if (not maintainer_files) and subdir:
+                    maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
+
+                maintainers = []
+                if readme_files:
+                    (desc, maintainers, deps) = readme_extract(readme_files[0])
+                    if desc:
+                        layer.summary = layer.name
+                        layer.description = desc
+                if maintainer_files:
+                    maintainers.extend(maintainers_extract(readme_files[0]))
+
+                if (not maintainers) and 'github.com' in layer.vcs_url:
+                    if json_data:
+                        layer.summary = json_data['description']
+                        layer.description = layer.summary
+                    if owner_json_data:
+                        owner_name = owner_json_data.get('name', None)
+                        owner_email = owner_json_data.get('email', None)
+                        if owner_name and owner_email:
+                            maintainers.append('%s <%s>' % (owner_name, owner_email))
+
+                if layer.name == 'openembedded-core':
+                    layer.summary = 'Core metadata'
+                    layer.layer_type = 'A'
+                elif layer.name == 'meta-oe':
+                    layer.summary = 'Additional shared OE metadata'
                     layer.description = layer.summary
-                if owner_json_data:
-                    owner_name = owner_json_data.get('name', None)
-                    owner_email = owner_json_data.get('email', None)
-                    if owner_name and owner_email:
-                        maintainers.append('%s <%s>' % (owner_name, owner_email))
-
-            if layer.name == 'openembedded-core':
-                layer.summary = 'Core metadata'
-                layer.layer_type = 'A'
-            elif layer.name == 'meta-oe':
-                layer.summary = 'Additional shared OE metadata'
-                layer.description = layer.summary
-                layer.layer_type = 'A'
-
-            if maintainers:
-                maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
-                for maintentry in maintainers:
-                    res = maint_re.match(maintentry)
-                    if res:
-                        maintainer = LayerMaintainer()
-                        maintainer.layerbranch = layerbranch
-                        maintainer.name = res.group(1).strip()
-                        maintainer.email = res.group(2)
-                        if res.group(3):
-                            maintainer.responsibility = res.group(3).strip()
-                        maintainer.save()
-
-            layer.save()
-
-        if options.dryrun:
-            transaction.rollback()
-        else:
-            transaction.commit()
-    except:
-        transaction.rollback()
-        raise
-    finally:
-        transaction.leave_transaction_management()
+                    layer.layer_type = 'A'
+
+                if maintainers:
+                    maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
+                    for maintentry in maintainers:
+                        res = maint_re.match(maintentry)
+                        if res:
+                            maintainer = LayerMaintainer()
+                            maintainer.layerbranch = layerbranch
+                            maintainer.name = res.group(1).strip()
+                            maintainer.email = res.group(2)
+                            if res.group(3):
+                                maintainer.responsibility = res.group(3).strip()
+                            maintainer.save()
+
+                layer.save()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
 
     sys.exit(0)