diff options
author | Alassane Yattara <alassane.yattara@savoirfairelinux.com> | 2023-10-04 14:44:15 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-10-05 21:17:09 +0100 |
commit | 2efb146480ee46c0463d9edb71bf1c03ce15bcf2 (patch) | |
tree | f2dcbccaf20ac8ff94555301ff5ae4d3f8c8fa2a /lib | |
parent | 868c88a249ef4b9fe5a891e76e25e054e4fcd994 (diff) | |
download | bitbake-contrib-2efb146480ee46c0463d9edb71bf1c03ce15bcf2.tar.gz |
toaster: Monitoring - implement Django logging system
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'lib')
-rw-r--r-- | lib/toaster/bldcollector/views.py | 3 | ||||
-rw-r--r-- | lib/toaster/logs/.gitignore | 1 | ||||
-rw-r--r-- | lib/toaster/toastergui/views.py | 7 | ||||
-rw-r--r-- | lib/toaster/toastergui/widgets.py | 4 | ||||
-rw-r--r-- | lib/toaster/toastermain/logs.py | 153 | ||||
-rw-r--r-- | lib/toaster/toastermain/settings.py | 66 | ||||
-rw-r--r-- | lib/toaster/toastermain/urls.py | 2 |
7 files changed, 198 insertions, 38 deletions
diff --git a/lib/toaster/bldcollector/views.py b/lib/toaster/bldcollector/views.py index 04cd8b3dd..bdf38ae6e 100644 --- a/lib/toaster/bldcollector/views.py +++ b/lib/toaster/bldcollector/views.py @@ -14,8 +14,11 @@ import subprocess import toastermain from django.views.decorators.csrf import csrf_exempt +from toastermain.logs import log_view_mixin + @csrf_exempt +@log_view_mixin def eventfile(request): """ Receives a file by POST, and runs toaster-eventreply on this file """ if request.method != "POST": diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore new file mode 100644 index 000000000..e5ebf25a4 --- /dev/null +++ b/lib/toaster/logs/.gitignore @@ -0,0 +1 @@ +*.log* diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 552ff1649..cc8517ba6 100644 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -34,6 +34,8 @@ import mimetypes import logging +from toastermain.logs import log_view_mixin + logger = logging.getLogger("toaster") # Project creation and managed build enable @@ -56,6 +58,7 @@ class MimeTypeFinder(object): return guessed_type # single point to add global values into the context before rendering +@log_view_mixin def toaster_render(request, page, context): context['project_enable'] = project_enable context['project_specific'] = is_project_specific @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): return response from django.http import HttpResponse +@log_view_mixin def xhr_dirinfo(request, build_id, target_id): top = request.GET.get('start', '/') return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") @@ -1612,6 +1616,7 @@ if True: from django.views.decorators.csrf import csrf_exempt @csrf_exempt + @log_view_mixin def xhr_testreleasechange(request, pid): def response(data): return HttpResponse(jsonfilter(data), @@ -1648,6 +1653,7 @@ if True: except Exception as e: return response({"error": str(e) }) + @log_view_mixin def xhr_configvaredit(request, pid): try: prj = Project.objects.get(id = pid) @@ -1726,6 +1732,7 @@ if True: return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") + @log_view_mixin def customrecipe_download(request, pid, recipe_id): recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) diff --git a/lib/toaster/toastergui/widgets.py b/lib/toaster/toastergui/widgets.py index 6e87285c8..b32abf40b 100644 --- a/lib/toaster/toastergui/widgets.py +++ b/lib/toaster/toastergui/widgets.py @@ -32,6 +32,7 @@ import re import os from toastergui.tablefilter import TableFilterMap +from toastermain.logs import log_view_mixin try: from urllib import unquote_plus @@ -84,6 +85,7 @@ class ToasterTable(TemplateView): return context + @log_view_mixin def get(self, request, *args, **kwargs): if request.GET.get('format', None) == 'json': @@ -415,6 +417,7 @@ class ToasterTypeAhead(View): def __init__(self, *args, **kwargs): super(ToasterTypeAhead, self).__init__() + @log_view_mixin def get(self, request, *args, **kwargs): def response(data): return HttpResponse(json.dumps(data, @@ -470,6 +473,7 @@ class MostRecentBuildsView(View): return False + @log_view_mixin def get(self, request, *args, **kwargs): """ Returns a list of builds in JSON format. diff --git a/lib/toaster/toastermain/logs.py b/lib/toaster/toastermain/logs.py new file mode 100644 index 000000000..f9953982b --- /dev/null +++ b/lib/toaster/toastermain/logs.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging +import json +from pathlib import Path +from django.http import HttpRequest + +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent + + +def log_api_request(request, response, view, logger_name='api'): + """Helper function for LogAPIMixin""" + + repjson = { + 'view': view, + 'path': request.path, + 'method': request.method, + 'status': response.status_code + } + + logger = logging.getLogger(logger_name) + logger.info( + json.dumps(repjson, indent=4, separators=(", ", " : ")) + ) + + +def log_view_mixin(view): + def log_view_request(*args, **kwargs): + # get request from args else kwargs + request = None + if len(args) > 0: + for req in args: + if isinstance(req, HttpRequest): + request = req + break + elif request is None: + request = kwargs.get('request') + + response = view(*args, **kwargs) + log_api_request( + request, response, request.resolver_match.view_name, 'toaster') + return response + return log_view_request + + + +class LogAPIMixin: + """Logs API requests + + tested with: + - APIView + - ModelViewSet + - ReadOnlyModelViewSet + - GenericAPIView + + Note: you can set `view_name` attribute in View to override get_view_name() + """ + + def get_view_name(self): + if hasattr(self, 'view_name'): + return self.view_name + return super().get_view_name() + + def finalize_response(self, request, response, *args, **kwargs): + log_api_request(request, response, self.get_view_name()) + return super().finalize_response(request, response, *args, **kwargs) + + +LOGGING_SETTINGS = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'datetime': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'verbose': { + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', + 'datefmt': "%d/%b/%Y %H:%M:%S", + 'style': '{', + }, + 'api': { + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', + 'style': '{' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'datetime', + }, + 'file_django': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/django.log', + 'when': 'D', # interval type + 'interval': 1, # defaults to 1 + 'backupCount': 10, # how many files to keep + 'formatter': 'verbose', + }, + 'file_api': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/api.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + 'file_toaster': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/toaster.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['file_django', 'console'], + 'level': 'WARN', + 'propagate': True, + }, + 'django': { + 'handlers': ['file_django', 'console'], + 'level': 'WARNING', + 'propogate': True, + }, + 'toaster': { + 'handlers': ['file_toaster'], + 'level': 'INFO', + 'propagate': False, + }, + 'api': { + 'handlers': ['file_api'], + 'level': 'INFO', + 'propagate': False, + } + } +} diff --git a/lib/toaster/toastermain/settings.py b/lib/toaster/toastermain/settings.py index 609c85d9d..b083cf588 100644 --- a/lib/toaster/toastermain/settings.py +++ b/lib/toaster/toastermain/settings.py @@ -9,6 +9,8 @@ # Django settings for Toaster project. import os +from pathlib import Path +from toastermain.logs import LOGGING_SETTINGS DEBUG = True @@ -186,7 +188,13 @@ TEMPLATES = [ 'django.template.loaders.app_directories.Loader', #'django.template.loaders.eggs.Loader', ], - 'string_if_invalid': InvalidString("%s"), + # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled + # Generally, string_if_invalid should only be enabled in order to debug + # a specific template problem, then cleared once debugging is complete. + # If you assign a value other than '' to string_if_invalid, + # you will experience rendering problems with these templates and sites. + # 'string_if_invalid': InvalidString("%s"), + 'string_if_invalid': "", 'debug': DEBUG, }, }, @@ -242,6 +250,9 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'bldcollector', 'toastermain', + + # 3rd-lib + "log_viewer", ) @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): # the site admins on every HTTP 500 error when DEBUG=False. # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - } - }, - 'formatters': { - 'datetime': { - 'format': '%(asctime)s %(levelname)s %(message)s' - } - }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'datetime', - } - }, - 'loggers': { - 'toaster' : { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'WARN', - 'propagate': True, - }, - } -} +LOGGING = LOGGING_SETTINGS + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent + +# LOG VIEWER +# https://pypi.org/project/django-log-viewer/ +LOG_VIEWER_FILES_PATTERN = '*.log*' +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] + +# Optionally you can set the next variables in order to customize the admin: +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" + if DEBUG and SQL_DEBUG: LOGGING['loggers']['django.db.backends'] = { diff --git a/lib/toaster/toastermain/urls.py b/lib/toaster/toastermain/urls.py index 036030266..3be46fcf0 100644 --- a/lib/toaster/toastermain/urls.py +++ b/lib/toaster/toastermain/urls.py @@ -28,6 +28,8 @@ urlpatterns = [ # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + url(r'^logs/', include('log_viewer.urls')), + # This is here to maintain backward compatibility and will be deprecated # in the future. url(r'^orm/eventfile$', bldcollector.views.eventfile), |