aboutsummaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster/toastergui/api.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/toaster/toastergui/api.py')
-rw-r--r--bitbake/lib/toaster/toastergui/api.py469
1 files changed, 459 insertions, 10 deletions
diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py
index 09fb02b8fc..be18090daf 100644
--- a/bitbake/lib/toaster/toastergui/api.py
+++ b/bitbake/lib/toaster/toastergui/api.py
@@ -16,21 +16,29 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# Please run flake8 on this file before sending patches
-# Temporary home for the UI's misc API
import re
+import logging
from orm.models import Project, ProjectTarget, Build, Layer_Version
from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
+from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
+from orm.models import Layer, Target, Package, Package_Dependency
from bldcontrol.models import BuildRequest
from bldcontrol import bbcontroller
+
from django.http import HttpResponse, JsonResponse
from django.views.generic import View
from django.core.urlresolvers import reverse
-from django.core import serializers
from django.utils import timezone
-from django.template.defaultfilters import date
+from django.db.models import Q, F
+from django.db import Error
from toastergui.templatetags.projecttags import json, sectohms, get_tasks
+from toastergui.templatetags.projecttags import filtered_filesizeformat
+
+logger = logging.getLogger("toaster")
+
def error_response(error):
return JsonResponse({"error": error})
@@ -216,6 +224,7 @@ class XhrLayer(View):
"redirect": reverse('project', args=(kwargs['pid'],))
})
+
class MostRecentBuildsView(View):
def _was_yesterday_or_earlier(self, completed_on):
now = timezone.now()
@@ -230,13 +239,11 @@ class MostRecentBuildsView(View):
"""
Returns a list of builds in JSON format.
"""
- mrb_type = 'all'
project = None
project_id = request.GET.get('project_id', None)
if project_id:
try:
- mrb_type = 'project'
project = Project.objects.get(pk=project_id)
except:
# if project lookup fails, assume no project
@@ -245,9 +252,6 @@ class MostRecentBuildsView(View):
recent_build_objs = Build.get_recent(project)
recent_builds = []
- # for timezone conversion
- tz = timezone.get_current_timezone()
-
for build_obj in recent_build_objs:
dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
buildtime_url = reverse('buildtime', args=(build_obj.pk,))
@@ -266,7 +270,8 @@ class MostRecentBuildsView(View):
build['buildrequest_id'] = buildrequest_id
build['recipes_parsed_percentage'] = \
- int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100)
+ int((build_obj.recipes_parsed /
+ build_obj.recipes_to_parse) * 100)
tasks_complete_percentage = 0
if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
@@ -300,7 +305,8 @@ class MostRecentBuildsView(View):
completed_on_template = '%H:%M'
if self._was_yesterday_or_earlier(completed_on):
completed_on_template = '%d/%m/%Y ' + completed_on_template
- build['completed_on'] = completed_on.strftime(completed_on_template)
+ build['completed_on'] = completed_on.strftime(
+ completed_on_template)
targets = []
target_objs = build_obj.get_sorted_target_list()
@@ -323,3 +329,446 @@ class MostRecentBuildsView(View):
recent_builds.append(build)
return JsonResponse(recent_builds, safe=False)
+
+
+class XhrCustomRecipe(View):
+ """ Create a custom image recipe """
+
+ def post(self, request, *args, **kwargs):
+ """
+ Custom image recipe REST API
+
+ Entry point: /xhr_customrecipe/
+ Method: POST
+
+ Args:
+ name: name of custom recipe to create
+ project: target project id of orm.models.Project
+ base: base recipe id of orm.models.Recipe
+
+ Returns:
+ {"error": "ok",
+ "url": <url of the created recipe>}
+ or
+ {"error": <error message>}
+ """
+ # check if request has all required parameters
+ for param in ('name', 'project', 'base'):
+ if param not in request.POST:
+ return error_response("Missing parameter '%s'" % param)
+
+ # get project and baserecipe objects
+ params = {}
+ for name, model in [("project", Project),
+ ("base", Recipe)]:
+ value = request.POST[name]
+ try:
+ params[name] = model.objects.get(id=value)
+ except model.DoesNotExist:
+ return error_response("Invalid %s id %s" % (name, value))
+
+ # create custom recipe
+ try:
+
+ # Only allowed chars in name are a-z, 0-9 and -
+ if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
+ return error_response("invalid-name")
+
+ custom_images = CustomImageRecipe.objects.all()
+
+ # Are there any recipes with this name already in our project?
+ existing_image_recipes_in_project = custom_images.filter(
+ name=request.POST["name"], project=params["project"])
+
+ if existing_image_recipes_in_project.count() > 0:
+ return error_response("image-already-exists")
+
+ # Are there any recipes with this name which aren't custom
+ # image recipes?
+ custom_image_ids = custom_images.values_list('id', flat=True)
+ existing_non_image_recipes = Recipe.objects.filter(
+ Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
+ )
+
+ if existing_non_image_recipes.count() > 0:
+ return error_response("recipe-already-exists")
+
+ # create layer 'Custom layer' and verion if needed
+ layer = Layer.objects.get_or_create(
+ name=CustomImageRecipe.LAYER_NAME,
+ summary="Layer for custom recipes",
+ vcs_url="file:///toaster_created_layer")[0]
+
+ # Check if we have a layer version already
+ # We don't use get_or_create here because the dirpath will change
+ # and is a required field
+ lver = Layer_Version.objects.filter(Q(project=params['project']) &
+ Q(layer=layer) &
+ Q(build=None)).last()
+ if lver is None:
+ lver, created = Layer_Version.objects.get_or_create(
+ project=params['project'],
+ layer=layer,
+ dirpath="toaster_created_layer")
+
+ # Add a dependency on our layer to the base recipe's layer
+ LayerVersionDependency.objects.get_or_create(
+ layer_version=lver,
+ depends_on=params["base"].layer_version)
+
+ # Add it to our current project if needed
+ ProjectLayer.objects.get_or_create(project=params['project'],
+ layercommit=lver,
+ optional=False)
+
+ # Create the actual recipe
+ recipe, created = CustomImageRecipe.objects.get_or_create(
+ name=request.POST["name"],
+ base_recipe=params["base"],
+ project=params["project"],
+ layer_version=lver,
+ is_image=True)
+
+ # If we created the object then setup these fields. They may get
+ # overwritten later on and cause the get_or_create to create a
+ # duplicate if they've changed.
+ if created:
+ recipe.file_path = request.POST["name"]
+ recipe.license = "MIT"
+ recipe.version = "0.1"
+ recipe.save()
+
+ except Error as err:
+ return error_response("Can't create custom recipe: %s" % err)
+
+ # Find the package list from the last build of this recipe/target
+ target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
+ Q(build__project=params['project']) &
+ (Q(target=params['base'].name) |
+ Q(target=recipe.name))).last()
+ if target:
+ # Copy in every package
+ # We don't want these packages to be linked to anything because
+ # that underlying data may change e.g. delete a build
+ for tpackage in target.target_installed_package_set.all():
+ try:
+ built_package = tpackage.package
+ # The package had no recipe information so is a ghost
+ # package skip it
+ if built_package.recipe is None:
+ continue
+
+ config_package = CustomImagePackage.objects.get(
+ name=built_package.name)
+
+ recipe.includes_set.add(config_package)
+ except Exception as e:
+ logger.warning("Error adding package %s %s" %
+ (tpackage.package.name, e))
+ pass
+
+ return JsonResponse(
+ {"error": "ok",
+ "packages": recipe.get_all_packages().count(),
+ "url": reverse('customrecipe', args=(params['project'].pk,
+ recipe.id))})
+
+
+class XhrCustomRecipeId(View):
+ """
+ Set of ReST API processors working with recipe id.
+
+ Entry point: /xhr_customrecipe/<recipe_id>
+
+ Methods:
+ GET - Get details of custom image recipe
+ DELETE - Delete custom image recipe
+
+ Returns:
+ GET:
+ {"error": "ok",
+ "info": dictionary of field name -> value pairs
+ of the CustomImageRecipe model}
+ DELETE:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ @staticmethod
+ def _get_ci_recipe(recipe_id):
+ """ Get Custom Image recipe or return an error response"""
+ try:
+ custom_recipe = \
+ CustomImageRecipe.objects.get(pk=recipe_id)
+ return custom_recipe, None
+
+ except CustomImageRecipe.DoesNotExist:
+ return None, error_response("Custom recipe with id=%s "
+ "not found" % recipe_id)
+
+ def get(self, request, *args, **kwargs):
+ custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+ if error:
+ return error
+
+ if request.method == 'GET':
+ info = {"id": custom_recipe.id,
+ "name": custom_recipe.name,
+ "base_recipe_id": custom_recipe.base_recipe.id,
+ "project_id": custom_recipe.project.id}
+
+ return JsonResponse({"error": "ok", "info": info})
+
+ def delete(self, request, *args, **kwargs):
+ custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+ if error:
+ return error
+
+ custom_recipe.delete()
+ return JsonResponse({"error": "ok"})
+
+
+class XhrCustomRecipePackages(View):
+ """
+ ReST API to add/remove packages to/from custom recipe.
+
+ Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
+ Methods:
+ PUT - Add package to the recipe
+ DELETE - Delete package from the recipe
+ GET - Get package information
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ @staticmethod
+ def _get_package(package_id):
+ try:
+ package = CustomImagePackage.objects.get(pk=package_id)
+ return package, None
+ except Package.DoesNotExist:
+ return None, error_response("Package with id=%s "
+ "not found" % package_id)
+
+ def _traverse_dependents(self, next_package_id,
+ rev_deps, all_current_packages, tree_level=0):
+ """
+ Recurse through reverse dependency tree for next_package_id.
+ Limit the reverse dependency search to packages not already scanned,
+ that is, not already in rev_deps.
+ Limit the scan to a depth (tree_level) not exceeding the count of
+ all packages in the custom image, and if that depth is exceeded
+ return False, pop out of the recursion, and write a warning
+ to the log, but this is unlikely, suggesting a dependency loop
+ not caught by bitbake.
+ On return, the input/output arg rev_deps is appended with queryset
+ dictionary elements, annotated for use in the customimage template.
+ The list has unsorted, but unique elements.
+ """
+ max_dependency_tree_depth = all_current_packages.count()
+ if tree_level >= max_dependency_tree_depth:
+ logger.warning(
+ "The number of reverse dependencies "
+ "for this package exceeds " + max_dependency_tree_depth +
+ " and the remaining reverse dependencies will not be removed")
+ return True
+
+ package = CustomImagePackage.objects.get(id=next_package_id)
+ dependents = \
+ package.package_dependencies_target.annotate(
+ name=F('package__name'),
+ pk=F('package__pk'),
+ size=F('package__size'),
+ ).values("name", "pk", "size").exclude(
+ ~Q(pk__in=all_current_packages)
+ )
+
+ for pkg in dependents:
+ if pkg in rev_deps:
+ # already seen, skip dependent search
+ continue
+
+ rev_deps.append(pkg)
+ if (self._traverse_dependents(pkg["pk"], rev_deps,
+ all_current_packages,
+ tree_level+1)):
+ return True
+
+ return False
+
+ def _get_all_dependents(self, package_id, all_current_packages):
+ """
+ Returns sorted list of recursive reverse dependencies for package_id,
+ as a list of dictionary items, by recursing through dependency
+ relationships.
+ """
+ rev_deps = []
+ self._traverse_dependents(package_id, rev_deps, all_current_packages)
+ rev_deps = sorted(rev_deps, key=lambda x: x["name"])
+ return rev_deps
+
+ def get(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(
+ kwargs['recipe_id'])
+ if error:
+ return error
+
+ # If no package_id then list all the current packages
+ if not kwargs['package_id']:
+ total_size = 0
+ packages = recipe.get_all_packages().values("id",
+ "name",
+ "version",
+ "size")
+ for package in packages:
+ package['size_formatted'] = \
+ filtered_filesizeformat(package['size'])
+ total_size += package['size']
+
+ return JsonResponse({"error": "ok",
+ "packages": list(packages),
+ "total": len(packages),
+ "total_size": total_size,
+ "total_size_formatted":
+ filtered_filesizeformat(total_size)})
+ else:
+ package, error = XhrCustomRecipePackages._get_package(
+ kwargs['package_id'])
+ if error:
+ return error
+
+ all_current_packages = recipe.get_all_packages()
+
+ # Dependencies for package which aren't satisfied by the
+ # current packages in the custom image recipe
+ deps = package.package_dependencies_source.for_target_or_none(
+ recipe.name)['packages'].annotate(
+ name=F('depends_on__name'),
+ pk=F('depends_on__pk'),
+ size=F('depends_on__size'),
+ ).values("name", "pk", "size").filter(
+ # There are two depends types we don't know why
+ (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
+ Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
+ ~Q(pk__in=all_current_packages)
+ )
+
+ # Reverse dependencies which are needed by packages that are
+ # in the image. Recursive search providing all dependents,
+ # not just immediate dependents.
+ reverse_deps = self._get_all_dependents(kwargs['package_id'],
+ all_current_packages)
+ total_size_deps = 0
+ total_size_reverse_deps = 0
+
+ for dep in deps:
+ dep['size_formatted'] = \
+ filtered_filesizeformat(dep['size'])
+ total_size_deps += dep['size']
+
+ for dep in reverse_deps:
+ dep['size_formatted'] = \
+ filtered_filesizeformat(dep['size'])
+ total_size_reverse_deps += dep['size']
+
+ return JsonResponse(
+ {"error": "ok",
+ "id": package.pk,
+ "name": package.name,
+ "version": package.version,
+ "unsatisfied_dependencies": list(deps),
+ "unsatisfied_dependencies_size": total_size_deps,
+ "unsatisfied_dependencies_size_formatted":
+ filtered_filesizeformat(total_size_deps),
+ "reverse_dependencies": list(reverse_deps),
+ "reverse_dependencies_size": total_size_reverse_deps,
+ "reverse_dependencies_size_formatted":
+ filtered_filesizeformat(total_size_reverse_deps)})
+
+ def put(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+ package, error = self._get_package(kwargs['package_id'])
+ if error:
+ return error
+
+ included_packages = recipe.includes_set.values_list('pk',
+ flat=True)
+
+ # If we're adding back a package which used to be included in this
+ # image all we need to do is remove it from the excludes
+ if package.pk in included_packages:
+ try:
+ recipe.excludes_set.remove(package)
+ return {"error": "ok"}
+ except Package.DoesNotExist:
+ return error_response("Package %s not found in excludes"
+ " but was in included list" %
+ package.name)
+
+ else:
+ recipe.appends_set.add(package)
+ # Make sure that package is not in the excludes set
+ try:
+ recipe.excludes_set.remove(package)
+ except:
+ pass
+ # Add the dependencies we think will be added to the recipe
+ # as a result of appending this package.
+ # TODO this should recurse down the entire deps tree
+ for dep in package.package_dependencies_source.all_depends():
+ try:
+ cust_package = CustomImagePackage.objects.get(
+ name=dep.depends_on.name)
+
+ recipe.includes_set.add(cust_package)
+ try:
+ # When adding the pre-requisite package, make
+ # sure it's not in the excluded list from a
+ # prior removal.
+ recipe.excludes_set.remove(cust_package)
+ except package.DoesNotExist:
+ # Don't care if the package had never been excluded
+ pass
+ except:
+ logger.warning("Could not add package's suggested"
+ "dependencies to the list")
+ return JsonResponse({"error": "ok"})
+
+ def delete(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+ package, error = self._get_package(kwargs['package_id'])
+ if error:
+ return error
+
+ try:
+ included_packages = recipe.includes_set.values_list('pk',
+ flat=True)
+ # If we're deleting a package which is included we need to
+ # Add it to the excludes list.
+ if package.pk in included_packages:
+ recipe.excludes_set.add(package)
+ else:
+ recipe.appends_set.remove(package)
+ all_current_packages = recipe.get_all_packages()
+
+ reverse_deps_dictlist = self._get_all_dependents(
+ package.pk,
+ all_current_packages)
+
+ ids = [entry['pk'] for entry in reverse_deps_dictlist]
+ reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
+ for r in reverse_deps:
+ try:
+ if r.id in included_packages:
+ recipe.excludes_set.add(r)
+ else:
+ recipe.appends_set.remove(r)
+ except:
+ pass
+
+ return JsonResponse({"error": "ok"})
+ except CustomImageRecipe.DoesNotExist:
+ return error_response("Tried to remove package that wasn't"
+ " present")