#!/usr/bin/env python

# Import a layer into the database
#
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details


import sys
import os.path

sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))

import optparse
import re
import glob
import utils
import logging
import subprocess

logger = utils.logger_create('LayerIndexImport')

link_re = re.compile(r'\[(http.*) +link\]')

def set_vcs_fields(layer, repoval):
    layer.vcs_url = repoval
    if repoval.startswith('git://git.openembedded.org/'):
        reponame = re.sub('^.*/', '', repoval)
        layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
        layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
        layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
    elif 'git.yoctoproject.org/' in repoval:
        reponame = re.sub('^.*/', '', repoval)
        layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
        layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
        layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
    elif 'github.com/' in repoval:
        reponame = re.sub('^.*github.com/', '', repoval)
        reponame = re.sub('.git$', '', reponame)
        layer.vcs_web_url = 'http://github.com/' + reponame
        layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
        layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
    elif 'gitorious.org/' in repoval:
        reponame = re.sub('^.*gitorious.org/', '', repoval)
        reponame = re.sub('.git$', '', reponame)
        layer.vcs_web_url = 'http://gitorious.org/' + reponame
        layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
        layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
    elif 'bitbucket.org/' in repoval:
        reponame = re.sub('^.*bitbucket.org/', '', repoval)
        reponame = re.sub('.git$', '', reponame)
        layer.vcs_web_url = 'http://bitbucket.org/' + reponame
        layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
        layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'


def readme_extract(readmefn):
    maintainer_re = re.compile('maintaine[r(s)ed by]*[:\n\r]', re.IGNORECASE)
    deps_re = re.compile('depend[sencies upon]*[:\n\r]', re.IGNORECASE)

    maintlines = []
    deps = []
    desc = ''
    maint_mode = False
    blank_seen = False
    deps_mode = False
    desc_mode = True
    with open(readmefn, 'r') as f:
        for line in f.readlines():
            if deps_mode:
                if maintainer_re.search(line):
                    deps_mode = False
                else:
                    if ':' in line:
                        blank_seen = False
                        if line.startswith('URI:'):
                            deps.append(line.split(':', 1)[-1].strip())
                        if line.startswith('layers:'):
                            deps[len(deps)-1] = (deps[len(deps)-1], line.split(':', 1)[-1].strip())
                    elif not (line.startswith('====') or line.startswith('----')):
                        if blank_seen:
                            deps_mode = False
                        else:
                            blank_seen = True
                    continue

            if maint_mode:
                line = line.strip()
                if line and '@' in line or ' at ' in line:
                    maintlines.append(line)
                elif not (line.startswith('====') or line.startswith('----')):
                    if maintlines or blank_seen:
                        maint_mode = False
                    else:
                        blank_seen = True
            elif maintainer_re.search(line):
                desc_mode = False
                maint_mode = True
                blank_seen = False
                if ':' in line:
                    line = line.rsplit(":", 1)[-1].strip()
                    if line:
                        maintlines.append(line)
            elif deps_re.search(line):
                desc_mode = False
                deps_mode = True
                blank_seen = False
            elif desc_mode:
                if not line.strip():
                    if blank_seen:
                        desc_mode = False
                    blank_seen = True
                elif line.startswith('====') or line.startswith('----'):
                    # Assume we just got the title, we don't need that
                    desc = ''
                else:
                    desc += line

    maintainers = []
    for line in maintlines:
        for maint in line.split(','):
            if '@' in maint or ' at ' in maint and not 'yyyyyy@zzzzz.com' in maint:
                maintainers.append(maint.strip())
    return desc, maintainers, deps


def maintainers_extract(maintfn):
    maintainers = []
    with open(maintfn, 'r') as f:
        for line in f.readlines():
            if line.startswith('M:'):
                line = line.split(':', 1)[-1].strip()
                if line and '@' in line or ' at ' in line:
                    maintainers.append(line)
    return list(set(maintainers))


def get_github_layerinfo(layer_url, username = None, password = None):
    import httplib
    import json
    from layerindex.models import LayerMaintainer

    def github_api_call(path):
        conn = httplib.HTTPSConnection('api.github.com')
        headers = {"User-Agent": "test_github.py"}
        if username:
            import base64
            auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
            headers['Authorization'] = "Basic %s" % auth

        conn.request("GET", path, headers=headers)
        resp = conn.getresponse()
        return resp

    json_data = None
    owner_json_data = None

    if layer_url.endswith('.git'):
        layer_url = layer_url[:-4]
    resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
    if resp.status in [200, 302]:
        data = resp.read()
        json_data = json.loads(data)
        #headers = dict((key, value) for key, value in resp.getheaders())
        #print headers
        owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
        if resp.status in [200, 302]:
            owner_data = owner_resp.read()
            owner_json_data = json.loads(owner_data)
        else:
            logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read()))
    else:
        logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read()))

    return (json_data, owner_json_data)


def main():
    parser = optparse.OptionParser(
        usage = """
    %prog [options] <url> [name]""")

    parser.add_option("-s", "--subdir",
            help = "Specify subdirectory",
            action="store", dest="subdir")
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("", "--github-auth",
            help = "Specify github username:password",
            action="store", dest="github_auth")
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")

    options, args = parser.parse_args(sys.argv)

    if len(args) < 2:
        print("Please specify URL of repository for layer")
        sys.exit(1)

    layer_url = args[1]

    if len(args) > 2:
        layer_name = args[2]
    else:
        if options.subdir:
            layer_name = options.subdir
        else:
            layer_name = filter(None, layer_url.split('/'))[-1]
            if layer_name.endswith('.git'):
                layer_name = layer_name[:-4]

    if options.github_auth:
        if not ':' in options.github_auth:
            logger.error('--github-auth value must be specified as username:password')
            sys.exit(1)
        splitval = options.github_auth.split(':')
        github_login = splitval[0]
        github_password = splitval[1]
    else:
        github_login = None
        github_password = None

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
    from django.db import transaction

    logger.setLevel(options.loglevel)

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)

    master_branch = utils.get_branch('master')
    core_layer = None
    transaction.enter_transaction_management()
    transaction.managed(True)
    try:
        # Fetch layer
        logger.info('Fetching repository %s' % layer_url)

        layer = LayerItem()
        layer.name = layer_name
        layer.status = 'P'
        layer.layer_type = 'M'
        layer.summary = 'tempvalue'
        layer.description = layer.summary

        set_vcs_fields(layer, layer_url)

        urldir = layer.get_fetch_dir()
        repodir = os.path.join(fetchdir, urldir)
        out = None
        try:
            if not os.path.exists(repodir):
                out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
            else:
                out = utils.runcmd("git fetch", repodir, logger=logger)
        except Exception as e:
            logger.error("Fetch failed: %s" % str(e))
            sys.exit(1)

        actual_branch = ''
        try:
            out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
        except subprocess.CalledProcessError:
            branches = utils.runcmd("git branch -r", repodir, logger=logger)
            for line in branches.splitlines():
                if 'origin/HEAD ->' in line:
                    actual_branch = line.split('-> origin/')[-1]
                    break
            if not actual_branch:
                logger.error("Repository has no master branch nor origin/HEAD")
                sys.exit(1)
            out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)

        layer_paths = []
        if options.subdir:
            layerdir = os.path.join(repodir, options.subdir)
            if not os.path.exists(layerdir):
                logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
                sys.exit(1)
            if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
                logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
                sys.exit(1)
            layer_paths.append(layerdir)
        else:
            if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
                layer_paths.append(repodir)
            # Find subdirs with a conf/layer.conf
            for subdir in os.listdir(repodir):
                subdir_path = os.path.join(repodir, subdir)
                if os.path.isdir(subdir_path):
                    if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
                        layer_paths.append(subdir_path)
            if not layer_paths:
                logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
                sys.exit(1)

        if 'github.com' in layer.vcs_url:
            json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)

        for layerdir in layer_paths:
            layer.pk = None
            if layerdir != repodir:
                subdir = os.path.relpath(layerdir, repodir)
                if len(layer_paths) > 1:
                    layer.name = subdir
            else:
                subdir = ''
            if LayerItem.objects.filter(name=layer.name).exists():
                logger.error('A layer named "%s" already exists in the database' % layer_name)
                sys.exit(1)

            logger.info('Creating layer %s' % layer.name)
            # Guess layer type
            if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
                layer.layer_type = 'D'
            elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
                layer.layer_type = 'B'
            layer.save()
            layerbranch = LayerBranch()
            layerbranch.layer = layer
            layerbranch.branch = master_branch
            if layerdir != repodir:
                layerbranch.vcs_subdir = subdir
            if actual_branch:
                layerbranch.actual_branch = actual_branch
            layerbranch.save()
            if layer.name != settings.CORE_LAYER_NAME:
                if not core_layer:
                    core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
                if core_layer:
                    layerdep = LayerDependency()
                    layerdep.layerbranch = layerbranch
                    layerdep.dependency = core_layer
                    layerdep.save()

            # Get some extra meta-information
            readme_files = glob.glob(os.path.join(layerdir, 'README*'))
            if (not readme_files) and subdir:
                readme_files = glob.glob(os.path.join(repodir, 'README*'))
            maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
            if (not maintainer_files) and subdir:
                maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))

            maintainers = []
            if readme_files:
                (desc, maintainers, deps) = readme_extract(readme_files[0])
                if desc:
                    layer.summary = layer.name
                    layer.description = desc
            if maintainer_files:
                maintainers.extend(maintainers_extract(readme_files[0]))

            if (not maintainers) and 'github.com' in layer.vcs_url:
                if json_data:
                    layer.summary = json_data['description']
                    layer.description = layer.summary
                if owner_json_data:
                    owner_name = owner_json_data.get('name', None)
                    owner_email = owner_json_data.get('email', None)
                    if owner_name and owner_email:
                        maintainers.append('%s <%s>' % (owner_name, owner_email))

            if layer.name == 'openembedded-core':
                layer.summary = 'Core metadata'
                layer.layer_type = 'A'
            elif layer.name == 'meta-oe':
                layer.summary = 'Additional shared OE metadata'
                layer.description = layer.summary
                layer.layer_type = 'A'

            if maintainers:
                maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.*)?$')
                for maintentry in maintainers:
                    res = maint_re.match(maintentry)
                    if res:
                        maintainer = LayerMaintainer()
                        maintainer.layerbranch = layerbranch
                        maintainer.name = res.group(1).strip()
                        maintainer.email = res.group(2)
                        maintainer.responsibility = res.group(3).strip()
                        maintainer.save()

            layer.save()

        if options.dryrun:
            transaction.rollback()
        else:
            transaction.commit()
    except:
        transaction.rollback()
        raise
    finally:
        transaction.leave_transaction_management()

    sys.exit(0)


if __name__ == "__main__":
    main()