| Message ID | 20231004134415.11070-1-alassane.yattara@savoirfairelinux.com |
|---|---|
| State | Accepted, archived |
| Commit | 2efb146480ee46c0463d9edb71bf1c03ce15bcf2 |
| Headers | show |
| Series | toaster: Monitoring - implement Django logging system | expand |
i think this introduces a missing dependency in toaster-requirements.txt on django-log-viewer or similar, as there is now a failure on https://github.com/crops/toaster-container for "master" 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 1014, in _gcd_import 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 991, in _find_and_load 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 973, in _find_and_load_unlocked 03:55:49 E: 0.862 ModuleNotFoundError: No module named 'log_viewer'" " On Wed, Oct 4, 2023 at 6:45 AM Alassane Yattara < alassane.yattara@savoirfairelinux.com> wrote: > --- > lib/toaster/bldcollector/views.py | 3 + > lib/toaster/logs/.gitignore | 1 + > lib/toaster/toastergui/views.py | 7 ++ > lib/toaster/toastergui/widgets.py | 4 + > lib/toaster/toastermain/logs.py | 153 ++++++++++++++++++++++++++++ > lib/toaster/toastermain/settings.py | 66 +++++------- > lib/toaster/toastermain/urls.py | 2 + > 7 files changed, 198 insertions(+), 38 deletions(-) > create mode 100644 lib/toaster/logs/.gitignore > create mode 100644 lib/toaster/toastermain/logs.py > > diff --git a/lib/toaster/bldcollector/views.py > b/lib/toaster/bldcollector/views.py > index 04cd8b3d..bdf38ae6 100644 > --- a/lib/toaster/bldcollector/views.py > +++ b/lib/toaster/bldcollector/views.py > @@ -14,8 +14,11 @@ import subprocess > import toastermain > from django.views.decorators.csrf import csrf_exempt > > +from toastermain.logs import log_view_mixin > + > > @csrf_exempt > +@log_view_mixin > def eventfile(request): > """ Receives a file by POST, and runs toaster-eventreply on this file > """ > if request.method != "POST": > diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore > new file mode 100644 > index 00000000..e5ebf25a > --- /dev/null > +++ b/lib/toaster/logs/.gitignore > @@ -0,0 +1 @@ > +*.log* > diff --git a/lib/toaster/toastergui/views.py > b/lib/toaster/toastergui/views.py > index 552ff164..cc8517ba 100644 > --- a/lib/toaster/toastergui/views.py > +++ b/lib/toaster/toastergui/views.py > @@ -34,6 +34,8 @@ import mimetypes > > import logging > > +from toastermain.logs import log_view_mixin > + > logger = logging.getLogger("toaster") > > # Project creation and managed build enable > @@ -56,6 +58,7 @@ class MimeTypeFinder(object): > return guessed_type > > # single point to add global values into the context before rendering > +@log_view_mixin > def toaster_render(request, page, context): > context['project_enable'] = project_enable > context['project_specific'] = is_project_specific > @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): > return response > > from django.http import HttpResponse > +@log_view_mixin > def xhr_dirinfo(request, build_id, target_id): > top = request.GET.get('start', '/') > return HttpResponse(_get_dir_entries(build_id, target_id, top), > content_type = "application/json") > @@ -1612,6 +1616,7 @@ if True: > > from django.views.decorators.csrf import csrf_exempt > @csrf_exempt > + @log_view_mixin > def xhr_testreleasechange(request, pid): > def response(data): > return HttpResponse(jsonfilter(data), > @@ -1648,6 +1653,7 @@ if True: > except Exception as e: > return response({"error": str(e) }) > > + @log_view_mixin > def xhr_configvaredit(request, pid): > try: > prj = Project.objects.get(id = pid) > @@ -1726,6 +1732,7 @@ if True: > return HttpResponse(json.dumps({"error":str(e) + "\n" + > traceback.format_exc()}), content_type = "application/json") > > > + @log_view_mixin > def customrecipe_download(request, pid, recipe_id): > recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) > > diff --git a/lib/toaster/toastergui/widgets.py > b/lib/toaster/toastergui/widgets.py > index 53696912..51ed153a 100644 > --- a/lib/toaster/toastergui/widgets.py > +++ b/lib/toaster/toastergui/widgets.py > @@ -32,6 +32,7 @@ import re > import os > > from toastergui.tablefilter import TableFilterMap > +from toastermain.logs import log_view_mixin > > try: > from urllib import unquote_plus > @@ -84,6 +85,7 @@ class ToasterTable(TemplateView): > > return context > > + @log_view_mixin > def get(self, request, *args, **kwargs): > if request.GET.get('format', None) == 'json': > > @@ -414,6 +416,7 @@ class ToasterTypeAhead(View): > def __init__(self, *args, **kwargs): > super(ToasterTypeAhead, self).__init__() > > + @log_view_mixin > def get(self, request, *args, **kwargs): > def response(data): > return HttpResponse(json.dumps(data, > @@ -469,6 +472,7 @@ class MostRecentBuildsView(View): > > return False > > + @log_view_mixin > def get(self, request, *args, **kwargs): > """ > Returns a list of builds in JSON format. > diff --git a/lib/toaster/toastermain/logs.py > b/lib/toaster/toastermain/logs.py > new file mode 100644 > index 00000000..f9953982 > --- /dev/null > +++ b/lib/toaster/toastermain/logs.py > @@ -0,0 +1,153 @@ > +#!/usr/bin/env python3 > +# -*- coding: utf-8 -*- > + > +import logging > +import json > +from pathlib import Path > +from django.http import HttpRequest > + > +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > + > + > +def log_api_request(request, response, view, logger_name='api'): > + """Helper function for LogAPIMixin""" > + > + repjson = { > + 'view': view, > + 'path': request.path, > + 'method': request.method, > + 'status': response.status_code > + } > + > + logger = logging.getLogger(logger_name) > + logger.info( > + json.dumps(repjson, indent=4, separators=(", ", " : ")) > + ) > + > + > +def log_view_mixin(view): > + def log_view_request(*args, **kwargs): > + # get request from args else kwargs > + request = None > + if len(args) > 0: > + for req in args: > + if isinstance(req, HttpRequest): > + request = req > + break > + elif request is None: > + request = kwargs.get('request') > + > + response = view(*args, **kwargs) > + log_api_request( > + request, response, request.resolver_match.view_name, > 'toaster') > + return response > + return log_view_request > + > + > + > +class LogAPIMixin: > + """Logs API requests > + > + tested with: > + - APIView > + - ModelViewSet > + - ReadOnlyModelViewSet > + - GenericAPIView > + > + Note: you can set `view_name` attribute in View to override > get_view_name() > + """ > + > + def get_view_name(self): > + if hasattr(self, 'view_name'): > + return self.view_name > + return super().get_view_name() > + > + def finalize_response(self, request, response, *args, **kwargs): > + log_api_request(request, response, self.get_view_name()) > + return super().finalize_response(request, response, *args, > **kwargs) > + > + > +LOGGING_SETTINGS = { > + 'version': 1, > + 'disable_existing_loggers': False, > + 'filters': { > + 'require_debug_false': { > + '()': 'django.utils.log.RequireDebugFalse' > + } > + }, > + 'formatters': { > + 'datetime': { > + 'format': '%(asctime)s %(levelname)s %(message)s' > + }, > + 'verbose': { > + 'format': '{levelname} {asctime} {module} {name}.{funcName} > {process:d} {thread:d} {message}', > + 'datefmt': "%d/%b/%Y %H:%M:%S", > + 'style': '{', > + }, > + 'api': { > + 'format': '\n{levelname} {asctime} > {name}.{funcName}:\n{message}', > + 'style': '{' > + } > + }, > + 'handlers': { > + 'mail_admins': { > + 'level': 'ERROR', > + 'filters': ['require_debug_false'], > + 'class': 'django.utils.log.AdminEmailHandler' > + }, > + 'console': { > + 'level': 'DEBUG', > + 'class': 'logging.StreamHandler', > + 'formatter': 'datetime', > + }, > + 'file_django': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/django.log', > + 'when': 'D', # interval type > + 'interval': 1, # defaults to 1 > + 'backupCount': 10, # how many files to keep > + 'formatter': 'verbose', > + }, > + 'file_api': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/api.log', > + 'when': 'D', > + 'interval': 1, > + 'backupCount': 10, > + 'formatter': 'verbose', > + }, > + 'file_toaster': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/toaster.log', > + 'when': 'D', > + 'interval': 1, > + 'backupCount': 10, > + 'formatter': 'verbose', > + }, > + }, > + 'loggers': { > + 'django.request': { > + 'handlers': ['file_django', 'console'], > + 'level': 'WARN', > + 'propagate': True, > + }, > + 'django': { > + 'handlers': ['file_django', 'console'], > + 'level': 'WARNING', > + 'propogate': True, > + }, > + 'toaster': { > + 'handlers': ['file_toaster'], > + 'level': 'INFO', > + 'propagate': False, > + }, > + 'api': { > + 'handlers': ['file_api'], > + 'level': 'INFO', > + 'propagate': False, > + } > + } > +} > diff --git a/lib/toaster/toastermain/settings.py > b/lib/toaster/toastermain/settings.py > index 609c85d9..b083cf58 100644 > --- a/lib/toaster/toastermain/settings.py > +++ b/lib/toaster/toastermain/settings.py > @@ -9,6 +9,8 @@ > # Django settings for Toaster project. > > import os > +from pathlib import Path > +from toastermain.logs import LOGGING_SETTINGS > > DEBUG = True > > @@ -186,7 +188,13 @@ TEMPLATES = [ > 'django.template.loaders.app_directories.Loader', > #'django.template.loaders.eggs.Loader', > ], > - 'string_if_invalid': InvalidString("%s"), > + # > https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled > + # Generally, string_if_invalid should only be enabled in > order to debug > + # a specific template problem, then cleared once debugging is > complete. > + # If you assign a value other than '' to string_if_invalid, > + # you will experience rendering problems with these templates > and sites. > + # 'string_if_invalid': InvalidString("%s"), > + 'string_if_invalid': "", > 'debug': DEBUG, > }, > }, > @@ -242,6 +250,9 @@ INSTALLED_APPS = ( > 'django.contrib.humanize', > 'bldcollector', > 'toastermain', > + > + # 3rd-lib > + "log_viewer", > ) > > > @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): > # the site admins on every HTTP 500 error when DEBUG=False. > # See http://docs.djangoproject.com/en/dev/topics/logging for > # more details on how to customize your logging configuration. > -LOGGING = { > - 'version': 1, > - 'disable_existing_loggers': False, > - 'filters': { > - 'require_debug_false': { > - '()': 'django.utils.log.RequireDebugFalse' > - } > - }, > - 'formatters': { > - 'datetime': { > - 'format': '%(asctime)s %(levelname)s %(message)s' > - } > - }, > - 'handlers': { > - 'mail_admins': { > - 'level': 'ERROR', > - 'filters': ['require_debug_false'], > - 'class': 'django.utils.log.AdminEmailHandler' > - }, > - 'console': { > - 'level': 'DEBUG', > - 'class': 'logging.StreamHandler', > - 'formatter': 'datetime', > - } > - }, > - 'loggers': { > - 'toaster' : { > - 'handlers': ['console'], > - 'level': 'DEBUG', > - }, > - 'django.request': { > - 'handlers': ['console'], > - 'level': 'WARN', > - 'propagate': True, > - }, > - } > -} > +LOGGING = LOGGING_SETTINGS > + > +# Build paths inside the project like this: BASE_DIR / 'subdir'. > +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > + > +# LOG VIEWER > +# https://pypi.org/project/django-log-viewer/ > +LOG_VIEWER_FILES_PATTERN = '*.log*' > +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') > +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page > +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read > +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] > + > +# Optionally you can set the next variables in order to customize the > admin: > +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" > + > > if DEBUG and SQL_DEBUG: > LOGGING['loggers']['django.db.backends'] = { > diff --git a/lib/toaster/toastermain/urls.py > b/lib/toaster/toastermain/urls.py > index 03603026..3be46fcf 100644 > --- a/lib/toaster/toastermain/urls.py > +++ b/lib/toaster/toastermain/urls.py > @@ -28,6 +28,8 @@ urlpatterns = [ > # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), > > > + url(r'^logs/', include('log_viewer.urls')), > + > # This is here to maintain backward compatibility and will be > deprecated > # in the future. > url(r'^orm/eventfile$', bldcollector.views.eventfile), > -- > 2.34.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15184): > https://lists.openembedded.org/g/bitbake-devel/message/15184 > Mute This Topic: https://lists.openembedded.org/mt/101755228/924729 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
Unfortunately, this commit has broken running in a container.
The system will start.
Traceback (most recent call last):
File "/usr/lib/python3.8/logging/config.py", line 563, in configure
handler = self.configure_handler(handlers[name])
File "/usr/lib/python3.8/logging/config.py", line 744, in
configure_handler
result = factory(**kwargs)
File "/usr/lib/python3.8/logging/handlers.py", line 200, in __init__
BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
File "/usr/lib/python3.8/logging/handlers.py", line 55, in __init__
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
File "/usr/lib/python3.8/logging/__init__.py", line 1147, in __init__
StreamHandler.__init__(self, self._open())
File "/usr/lib/python3.8/logging/__init__.py", line 1176, in _open
return open(self.baseFilename, self.mode, encoding=self.encoding)
PermissionError: [Errno 13] Permission denied:
'/home/usersetup/poky/bitbake/lib/toaster/logs/api.log'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/usersetup/poky/bitbake/bin/../lib/toaster/manage.py", line
16, in <module>
execute_from_command_line(sys.argv)
File
"/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py",
line 442, in execute_from_command_line
utility.execute()
File
"/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py",
line 416, in execute
django.setup()
File "/opt/venv/lib/python3.8/site-packages/django/__init__.py", line 19,
in setup
configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
File "/opt/venv/lib/python3.8/site-packages/django/utils/log.py", line
76, in configure_logging
logging_config_func(logging_settings)
File "/usr/lib/python3.8/logging/config.py", line 808, in dictConfig
dictConfigClass(config).configure()
File "/usr/lib/python3.8/logging/config.py", line 570, in configure
raise ValueError('Unable to configure handler '
ValueError: Unable to configure handler 'file_api'
The reason is that BASE_DIR is resolving to bitbake/lib/toaster and the
logs are in this case are attempting to write to a read only file system.
When running locally, this works, but perhaps is not where OpenEmbedded
users expect the logs to be:
$ ls bitbake/lib/toaster/logs
api.log toaster.log.2023-10-13 toaster.log.2023-10-16
django.log toaster.log.2023-10-14 toaster.log.2023-10-17
toaster.log toaster.log.2023-10-15
Previously, the logs were written into the build directory, like the
toaster_ui.log still is:
build-toaster-2/toaster_ui.log
This also pointed out an issue with the toaster script:
https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308
When we have a Python trace back, the code is not catching that there was a
failure to fully start nor fail.
e.g. "Successful start." was not output, but neither was "Toaster build
server not started."
The health check in
https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91
never sees "Successful start." so unfortunately, the test stage just
eventually times out.
On Wed, Oct 11, 2023 at 9:35 PM Tim Orling via lists.openembedded.org
<ticotimo=gmail.com@lists.openembedded.org> wrote:
> i think this introduces a missing dependency in toaster-requirements.txt
> on django-log-viewer or similar, as there is now a failure on
> https://github.com/crops/toaster-container for "master"
> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 1014, in
> _gcd_import
> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 991, in
> _find_and_load
> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 973, in
> _find_and_load_unlocked
> 03:55:49 E: 0.862 ModuleNotFoundError: No module named 'log_viewer'"
> "
>
>
> On Wed, Oct 4, 2023 at 6:45 AM Alassane Yattara <
> alassane.yattara@savoirfairelinux.com> wrote:
>
>> ---
>> lib/toaster/bldcollector/views.py | 3 +
>> lib/toaster/logs/.gitignore | 1 +
>> lib/toaster/toastergui/views.py | 7 ++
>> lib/toaster/toastergui/widgets.py | 4 +
>> lib/toaster/toastermain/logs.py | 153 ++++++++++++++++++++++++++++
>> lib/toaster/toastermain/settings.py | 66 +++++-------
>> lib/toaster/toastermain/urls.py | 2 +
>> 7 files changed, 198 insertions(+), 38 deletions(-)
>> create mode 100644 lib/toaster/logs/.gitignore
>> create mode 100644 lib/toaster/toastermain/logs.py
>>
>> diff --git a/lib/toaster/bldcollector/views.py
>> b/lib/toaster/bldcollector/views.py
>> index 04cd8b3d..bdf38ae6 100644
>> --- a/lib/toaster/bldcollector/views.py
>> +++ b/lib/toaster/bldcollector/views.py
>> @@ -14,8 +14,11 @@ import subprocess
>> import toastermain
>> from django.views.decorators.csrf import csrf_exempt
>>
>> +from toastermain.logs import log_view_mixin
>> +
>>
>> @csrf_exempt
>> +@log_view_mixin
>> def eventfile(request):
>> """ Receives a file by POST, and runs toaster-eventreply on this
>> file """
>> if request.method != "POST":
>> diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore
>> new file mode 100644
>> index 00000000..e5ebf25a
>> --- /dev/null
>> +++ b/lib/toaster/logs/.gitignore
>> @@ -0,0 +1 @@
>> +*.log*
>> diff --git a/lib/toaster/toastergui/views.py
>> b/lib/toaster/toastergui/views.py
>> index 552ff164..cc8517ba 100644
>> --- a/lib/toaster/toastergui/views.py
>> +++ b/lib/toaster/toastergui/views.py
>> @@ -34,6 +34,8 @@ import mimetypes
>>
>> import logging
>>
>> +from toastermain.logs import log_view_mixin
>> +
>> logger = logging.getLogger("toaster")
>>
>> # Project creation and managed build enable
>> @@ -56,6 +58,7 @@ class MimeTypeFinder(object):
>> return guessed_type
>>
>> # single point to add global values into the context before rendering
>> +@log_view_mixin
>> def toaster_render(request, page, context):
>> context['project_enable'] = project_enable
>> context['project_specific'] = is_project_specific
>> @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id):
>> return response
>>
>> from django.http import HttpResponse
>> +@log_view_mixin
>> def xhr_dirinfo(request, build_id, target_id):
>> top = request.GET.get('start', '/')
>> return HttpResponse(_get_dir_entries(build_id, target_id, top),
>> content_type = "application/json")
>> @@ -1612,6 +1616,7 @@ if True:
>>
>> from django.views.decorators.csrf import csrf_exempt
>> @csrf_exempt
>> + @log_view_mixin
>> def xhr_testreleasechange(request, pid):
>> def response(data):
>> return HttpResponse(jsonfilter(data),
>> @@ -1648,6 +1653,7 @@ if True:
>> except Exception as e:
>> return response({"error": str(e) })
>>
>> + @log_view_mixin
>> def xhr_configvaredit(request, pid):
>> try:
>> prj = Project.objects.get(id = pid)
>> @@ -1726,6 +1732,7 @@ if True:
>> return HttpResponse(json.dumps({"error":str(e) + "\n" +
>> traceback.format_exc()}), content_type = "application/json")
>>
>>
>> + @log_view_mixin
>> def customrecipe_download(request, pid, recipe_id):
>> recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
>>
>> diff --git a/lib/toaster/toastergui/widgets.py
>> b/lib/toaster/toastergui/widgets.py
>> index 53696912..51ed153a 100644
>> --- a/lib/toaster/toastergui/widgets.py
>> +++ b/lib/toaster/toastergui/widgets.py
>> @@ -32,6 +32,7 @@ import re
>> import os
>>
>> from toastergui.tablefilter import TableFilterMap
>> +from toastermain.logs import log_view_mixin
>>
>> try:
>> from urllib import unquote_plus
>> @@ -84,6 +85,7 @@ class ToasterTable(TemplateView):
>>
>> return context
>>
>> + @log_view_mixin
>> def get(self, request, *args, **kwargs):
>> if request.GET.get('format', None) == 'json':
>>
>> @@ -414,6 +416,7 @@ class ToasterTypeAhead(View):
>> def __init__(self, *args, **kwargs):
>> super(ToasterTypeAhead, self).__init__()
>>
>> + @log_view_mixin
>> def get(self, request, *args, **kwargs):
>> def response(data):
>> return HttpResponse(json.dumps(data,
>> @@ -469,6 +472,7 @@ class MostRecentBuildsView(View):
>>
>> return False
>>
>> + @log_view_mixin
>> def get(self, request, *args, **kwargs):
>> """
>> Returns a list of builds in JSON format.
>> diff --git a/lib/toaster/toastermain/logs.py
>> b/lib/toaster/toastermain/logs.py
>> new file mode 100644
>> index 00000000..f9953982
>> --- /dev/null
>> +++ b/lib/toaster/toastermain/logs.py
>> @@ -0,0 +1,153 @@
>> +#!/usr/bin/env python3
>> +# -*- coding: utf-8 -*-
>> +
>> +import logging
>> +import json
>> +from pathlib import Path
>> +from django.http import HttpRequest
>> +
>> +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
>> +
>> +
>> +def log_api_request(request, response, view, logger_name='api'):
>> + """Helper function for LogAPIMixin"""
>> +
>> + repjson = {
>> + 'view': view,
>> + 'path': request.path,
>> + 'method': request.method,
>> + 'status': response.status_code
>> + }
>> +
>> + logger = logging.getLogger(logger_name)
>> + logger.info(
>> + json.dumps(repjson, indent=4, separators=(", ", " : "))
>> + )
>> +
>> +
>> +def log_view_mixin(view):
>> + def log_view_request(*args, **kwargs):
>> + # get request from args else kwargs
>> + request = None
>> + if len(args) > 0:
>> + for req in args:
>> + if isinstance(req, HttpRequest):
>> + request = req
>> + break
>> + elif request is None:
>> + request = kwargs.get('request')
>> +
>> + response = view(*args, **kwargs)
>> + log_api_request(
>> + request, response, request.resolver_match.view_name,
>> 'toaster')
>> + return response
>> + return log_view_request
>> +
>> +
>> +
>> +class LogAPIMixin:
>> + """Logs API requests
>> +
>> + tested with:
>> + - APIView
>> + - ModelViewSet
>> + - ReadOnlyModelViewSet
>> + - GenericAPIView
>> +
>> + Note: you can set `view_name` attribute in View to override
>> get_view_name()
>> + """
>> +
>> + def get_view_name(self):
>> + if hasattr(self, 'view_name'):
>> + return self.view_name
>> + return super().get_view_name()
>> +
>> + def finalize_response(self, request, response, *args, **kwargs):
>> + log_api_request(request, response, self.get_view_name())
>> + return super().finalize_response(request, response, *args,
>> **kwargs)
>> +
>> +
>> +LOGGING_SETTINGS = {
>> + 'version': 1,
>> + 'disable_existing_loggers': False,
>> + 'filters': {
>> + 'require_debug_false': {
>> + '()': 'django.utils.log.RequireDebugFalse'
>> + }
>> + },
>> + 'formatters': {
>> + 'datetime': {
>> + 'format': '%(asctime)s %(levelname)s %(message)s'
>> + },
>> + 'verbose': {
>> + 'format': '{levelname} {asctime} {module} {name}.{funcName}
>> {process:d} {thread:d} {message}',
>> + 'datefmt': "%d/%b/%Y %H:%M:%S",
>> + 'style': '{',
>> + },
>> + 'api': {
>> + 'format': '\n{levelname} {asctime}
>> {name}.{funcName}:\n{message}',
>> + 'style': '{'
>> + }
>> + },
>> + 'handlers': {
>> + 'mail_admins': {
>> + 'level': 'ERROR',
>> + 'filters': ['require_debug_false'],
>> + 'class': 'django.utils.log.AdminEmailHandler'
>> + },
>> + 'console': {
>> + 'level': 'DEBUG',
>> + 'class': 'logging.StreamHandler',
>> + 'formatter': 'datetime',
>> + },
>> + 'file_django': {
>> + 'level': 'INFO',
>> + 'class': 'logging.handlers.TimedRotatingFileHandler',
>> + 'filename': BASE_DIR / 'logs/django.log',
>> + 'when': 'D', # interval type
>> + 'interval': 1, # defaults to 1
>> + 'backupCount': 10, # how many files to keep
>> + 'formatter': 'verbose',
>> + },
>> + 'file_api': {
>> + 'level': 'INFO',
>> + 'class': 'logging.handlers.TimedRotatingFileHandler',
>> + 'filename': BASE_DIR / 'logs/api.log',
>> + 'when': 'D',
>> + 'interval': 1,
>> + 'backupCount': 10,
>> + 'formatter': 'verbose',
>> + },
>> + 'file_toaster': {
>> + 'level': 'INFO',
>> + 'class': 'logging.handlers.TimedRotatingFileHandler',
>> + 'filename': BASE_DIR / 'logs/toaster.log',
>> + 'when': 'D',
>> + 'interval': 1,
>> + 'backupCount': 10,
>> + 'formatter': 'verbose',
>> + },
>> + },
>> + 'loggers': {
>> + 'django.request': {
>> + 'handlers': ['file_django', 'console'],
>> + 'level': 'WARN',
>> + 'propagate': True,
>> + },
>> + 'django': {
>> + 'handlers': ['file_django', 'console'],
>> + 'level': 'WARNING',
>> + 'propogate': True,
>> + },
>> + 'toaster': {
>> + 'handlers': ['file_toaster'],
>> + 'level': 'INFO',
>> + 'propagate': False,
>> + },
>> + 'api': {
>> + 'handlers': ['file_api'],
>> + 'level': 'INFO',
>> + 'propagate': False,
>> + }
>> + }
>> +}
>> diff --git a/lib/toaster/toastermain/settings.py
>> b/lib/toaster/toastermain/settings.py
>> index 609c85d9..b083cf58 100644
>> --- a/lib/toaster/toastermain/settings.py
>> +++ b/lib/toaster/toastermain/settings.py
>> @@ -9,6 +9,8 @@
>> # Django settings for Toaster project.
>>
>> import os
>> +from pathlib import Path
>> +from toastermain.logs import LOGGING_SETTINGS
>>
>> DEBUG = True
>>
>> @@ -186,7 +188,13 @@ TEMPLATES = [
>> 'django.template.loaders.app_directories.Loader',
>> #'django.template.loaders.eggs.Loader',
>> ],
>> - 'string_if_invalid': InvalidString("%s"),
>> + #
>> https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
>> + # Generally, string_if_invalid should only be enabled in
>> order to debug
>> + # a specific template problem, then cleared once debugging
>> is complete.
>> + # If you assign a value other than '' to string_if_invalid,
>> + # you will experience rendering problems with these
>> templates and sites.
>> + # 'string_if_invalid': InvalidString("%s"),
>> + 'string_if_invalid': "",
>> 'debug': DEBUG,
>> },
>> },
>> @@ -242,6 +250,9 @@ INSTALLED_APPS = (
>> 'django.contrib.humanize',
>> 'bldcollector',
>> 'toastermain',
>> +
>> + # 3rd-lib
>> + "log_viewer",
>> )
>>
>>
>> @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)):
>> # the site admins on every HTTP 500 error when DEBUG=False.
>> # See http://docs.djangoproject.com/en/dev/topics/logging for
>> # more details on how to customize your logging configuration.
>> -LOGGING = {
>> - 'version': 1,
>> - 'disable_existing_loggers': False,
>> - 'filters': {
>> - 'require_debug_false': {
>> - '()': 'django.utils.log.RequireDebugFalse'
>> - }
>> - },
>> - 'formatters': {
>> - 'datetime': {
>> - 'format': '%(asctime)s %(levelname)s %(message)s'
>> - }
>> - },
>> - 'handlers': {
>> - 'mail_admins': {
>> - 'level': 'ERROR',
>> - 'filters': ['require_debug_false'],
>> - 'class': 'django.utils.log.AdminEmailHandler'
>> - },
>> - 'console': {
>> - 'level': 'DEBUG',
>> - 'class': 'logging.StreamHandler',
>> - 'formatter': 'datetime',
>> - }
>> - },
>> - 'loggers': {
>> - 'toaster' : {
>> - 'handlers': ['console'],
>> - 'level': 'DEBUG',
>> - },
>> - 'django.request': {
>> - 'handlers': ['console'],
>> - 'level': 'WARN',
>> - 'propagate': True,
>> - },
>> - }
>> -}
>> +LOGGING = LOGGING_SETTINGS
>> +
>> +# Build paths inside the project like this: BASE_DIR / 'subdir'.
>> +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
>> +
>> +# LOG VIEWER
>> +# https://pypi.org/project/django-log-viewer/
>> +LOG_VIEWER_FILES_PATTERN = '*.log*'
>> +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs')
>> +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page
>> +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read
>> +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
>> +
>> +# Optionally you can set the next variables in order to customize the
>> admin:
>> +LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
>> +
>>
>> if DEBUG and SQL_DEBUG:
>> LOGGING['loggers']['django.db.backends'] = {
>> diff --git a/lib/toaster/toastermain/urls.py
>> b/lib/toaster/toastermain/urls.py
>> index 03603026..3be46fcf 100644
>> --- a/lib/toaster/toastermain/urls.py
>> +++ b/lib/toaster/toastermain/urls.py
>> @@ -28,6 +28,8 @@ urlpatterns = [
>> # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
>>
>>
>> + url(r'^logs/', include('log_viewer.urls')),
>> +
>> # This is here to maintain backward compatibility and will be
>> deprecated
>> # in the future.
>> url(r'^orm/eventfile$', bldcollector.views.eventfile),
>> --
>> 2.34.1
>>
>>
>>
>>
>>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#15212):
> https://lists.openembedded.org/g/bitbake-devel/message/15212
> Mute This Topic: https://lists.openembedded.org/mt/101755228/924729
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [
> ticotimo@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
>
Tim,
Seem OpenEmbedded user has not permission to log into bitbake/lib/toaster/logs,
Yes, I can use BUILDDIR instead of BASE_DIR as logging directory, but there is no way to know if that work in container.
I'm trying to reproduce the issue, i cloned and run repos https://github.com/crops/toaster-container master,
this pull crops/toaster:master, but I don't see logging system changes there.
Question:
- Is a way to pull a specific docker crops/toaster image that contain logging system changes ?
Alassane
----- Mail original -----
De: "Tim Orling" <ticotimo@gmail.com>
À: "Alassane Yattara" <alassane.yattara@savoirfairelinux.com>
Cc: bitbake-devel@lists.openembedded.org, toaster@lists.yoctoproject.org
Envoyé: Jeudi 19 Octobre 2023 15:00:35
Objet: Re: [Toaster] [bitbake-devel] [PATCH] toaster: Monitoring - implement Django logging system
Unfortunately, this commit has broken running in a container.
The system will start.
Traceback (most recent call last):
File "/usr/lib/python3.8/logging/config.py", line 563, in configure
handler = self.configure_handler(handlers[name])
File "/usr/lib/python3.8/logging/config.py", line 744, in configure_handler
result = factory(**kwargs)
File "/usr/lib/python3.8/logging/handlers.py", line 200, in __init__
BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
File "/usr/lib/python3.8/logging/handlers.py", line 55, in __init__
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
File "/usr/lib/python3.8/logging/__init__.py", line 1147, in __init__
StreamHandler.__init__(self, self._open())
File "/usr/lib/python3.8/logging/__init__.py", line 1176, in _open
return open(self.baseFilename, self.mode, encoding=self.encoding)
PermissionError: [Errno 13] Permission denied: '/home/usersetup/poky/bitbake/lib/toaster/logs/api.log'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/usersetup/poky/bitbake/bin/../lib/toaster/manage.py", line 16, in <module>
execute_from_command_line(sys.argv)
File "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", line 442, in execute_from_command_line
utility.execute()
File "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", line 416, in execute
django.setup()
File "/opt/venv/lib/python3.8/site-packages/django/__init__.py", line 19, in setup
configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)
File "/opt/venv/lib/python3.8/site-packages/django/utils/log.py", line 76, in configure_logging
logging_config_func(logging_settings)
File "/usr/lib/python3.8/logging/config.py", line 808, in dictConfig
dictConfigClass(config).configure()
File "/usr/lib/python3.8/logging/config.py", line 570, in configure
raise ValueError('Unable to configure handler '
ValueError: Unable to configure handler 'file_api'
The reason is that BASE_DIR is resolving to bitbake/lib/toaster and the logs are in this case are attempting to write to a read only file system.
When running locally, this works, but perhaps is not where OpenEmbedded users expect the logs to be:
$ ls bitbake/lib/toaster/logs
api.log toaster.log.2023-10-13 toaster.log.2023-10-16
django.log toaster.log.2023-10-14 toaster.log.2023-10-17
toaster.log toaster.log.2023-10-15
Previously, the logs were written into the build directory, like the toaster_ui.log still is:
build-toaster-2/toaster_ui.log
This also pointed out an issue with the toaster script:
[ https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 | https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 ]
When we have a Python trace back, the code is not catching that there was a failure to fully start nor fail.
e.g. "Successful start." was not output, but neither was "Toaster build server not started."
The health check in [ https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 | https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 ] never sees "Successful start." so unfortunately, the test stage just eventually times out.
On Wed, Oct 11, 2023 at 9:35 PM Tim Orling via [ http://lists.openembedded.org/ | lists.openembedded.org ] <ticotimo= [ mailto:gmail.com@lists.openembedded.org | gmail.com@lists.openembedded.org ] > wrote:
i think this introduces a missing dependency in toaster-requirements.txt on django-log-viewer or similar, as there is now a failure on [ https://github.com/crops/toaster-container | https://github.com/crops/toaster-container ] for "master"
03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 991, in _find_and_load
03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 973, in _find_and_load_unlocked
03:55:49 E: 0.862 ModuleNotFoundError: No module named 'log_viewer'"
"
On Wed, Oct 4, 2023 at 6:45 AM Alassane Yattara < [ mailto:alassane.yattara@savoirfairelinux.com | alassane.yattara@savoirfairelinux.com ] > wrote:
---
lib/toaster/bldcollector/views.py | 3 +
lib/toaster/logs/.gitignore | 1 +
lib/toaster/toastergui/views.py | 7 ++
lib/toaster/toastergui/widgets.py | 4 +
lib/toaster/toastermain/logs.py | 153 ++++++++++++++++++++++++++++
lib/toaster/toastermain/settings.py | 66 +++++-------
lib/toaster/toastermain/urls.py | 2 +
7 files changed, 198 insertions(+), 38 deletions(-)
create mode 100644 lib/toaster/logs/.gitignore
create mode 100644 lib/toaster/toastermain/logs.py
diff --git a/lib/toaster/bldcollector/views.py b/lib/toaster/bldcollector/views.py
index 04cd8b3d..bdf38ae6 100644
--- a/lib/toaster/bldcollector/views.py
+++ b/lib/toaster/bldcollector/views.py
@@ -14,8 +14,11 @@ import subprocess
import toastermain
from django.views.decorators.csrf import csrf_exempt
+from toastermain.logs import log_view_mixin
+
@csrf_exempt
+@log_view_mixin
def eventfile(request):
""" Receives a file by POST, and runs toaster-eventreply on this file """
if request.method != "POST":
diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore
new file mode 100644
index 00000000..e5ebf25a
--- /dev/null
+++ b/lib/toaster/logs/.gitignore
@@ -0,0 +1 @@
+*.log*
diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py
index 552ff164..cc8517ba 100644
--- a/lib/toaster/toastergui/views.py
+++ b/lib/toaster/toastergui/views.py
@@ -34,6 +34,8 @@ import mimetypes
import logging
+from toastermain.logs import log_view_mixin
+
logger = logging.getLogger("toaster")
# Project creation and managed build enable
@@ -56,6 +58,7 @@ class MimeTypeFinder(object):
return guessed_type
# single point to add global values into the context before rendering
+@log_view_mixin
def toaster_render(request, page, context):
context['project_enable'] = project_enable
context['project_specific'] = is_project_specific
@@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id):
return response
from django.http import HttpResponse
+@log_view_mixin
def xhr_dirinfo(request, build_id, target_id):
top = request.GET.get('start', '/')
return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json")
@@ -1612,6 +1616,7 @@ if True:
from django.views.decorators.csrf import csrf_exempt
@csrf_exempt
+ @log_view_mixin
def xhr_testreleasechange(request, pid):
def response(data):
return HttpResponse(jsonfilter(data),
@@ -1648,6 +1653,7 @@ if True:
except Exception as e:
return response({"error": str(e) })
+ @log_view_mixin
def xhr_configvaredit(request, pid):
try:
prj = Project.objects.get(id = pid)
@@ -1726,6 +1732,7 @@ if True:
return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
+ @log_view_mixin
def customrecipe_download(request, pid, recipe_id):
recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
diff --git a/lib/toaster/toastergui/widgets.py b/lib/toaster/toastergui/widgets.py
index 53696912..51ed153a 100644
--- a/lib/toaster/toastergui/widgets.py
+++ b/lib/toaster/toastergui/widgets.py
@@ -32,6 +32,7 @@ import re
import os
from toastergui.tablefilter import TableFilterMap
+from toastermain.logs import log_view_mixin
try:
from urllib import unquote_plus
@@ -84,6 +85,7 @@ class ToasterTable(TemplateView):
return context
+ @log_view_mixin
def get(self, request, *args, **kwargs):
if request.GET.get('format', None) == 'json':
@@ -414,6 +416,7 @@ class ToasterTypeAhead(View):
def __init__(self, *args, **kwargs):
super(ToasterTypeAhead, self).__init__()
+ @log_view_mixin
def get(self, request, *args, **kwargs):
def response(data):
return HttpResponse(json.dumps(data,
@@ -469,6 +472,7 @@ class MostRecentBuildsView(View):
return False
+ @log_view_mixin
def get(self, request, *args, **kwargs):
"""
Returns a list of builds in JSON format.
diff --git a/lib/toaster/toastermain/logs.py b/lib/toaster/toastermain/logs.py
new file mode 100644
index 00000000..f9953982
--- /dev/null
+++ b/lib/toaster/toastermain/logs.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import logging
+import json
+from pathlib import Path
+from django.http import HttpRequest
+
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
+
+
+def log_api_request(request, response, view, logger_name='api'):
+ """Helper function for LogAPIMixin"""
+
+ repjson = {
+ 'view': view,
+ 'path': request.path,
+ 'method': request.method,
+ 'status': response.status_code
+ }
+
+ logger = logging.getLogger(logger_name)
+ [ http://logger.info/ | logger.info ] (
+ json.dumps(repjson, indent=4, separators=(", ", " : "))
+ )
+
+
+def log_view_mixin(view):
+ def log_view_request(*args, **kwargs):
+ # get request from args else kwargs
+ request = None
+ if len(args) > 0:
+ for req in args:
+ if isinstance(req, HttpRequest):
+ request = req
+ break
+ elif request is None:
+ request = kwargs.get('request')
+
+ response = view(*args, **kwargs)
+ log_api_request(
+ request, response, request.resolver_match.view_name, 'toaster')
+ return response
+ return log_view_request
+
+
+
+class LogAPIMixin:
+ """Logs API requests
+
+ tested with:
+ - APIView
+ - ModelViewSet
+ - ReadOnlyModelViewSet
+ - GenericAPIView
+
+ Note: you can set `view_name` attribute in View to override get_view_name()
+ """
+
+ def get_view_name(self):
+ if hasattr(self, 'view_name'):
+ return self.view_name
+ return super().get_view_name()
+
+ def finalize_response(self, request, response, *args, **kwargs):
+ log_api_request(request, response, self.get_view_name())
+ return super().finalize_response(request, response, *args, **kwargs)
+
+
+LOGGING_SETTINGS = {
+ 'version': 1,
+ 'disable_existing_loggers': False,
+ 'filters': {
+ 'require_debug_false': {
+ '()': 'django.utils.log.RequireDebugFalse'
+ }
+ },
+ 'formatters': {
+ 'datetime': {
+ 'format': '%(asctime)s %(levelname)s %(message)s'
+ },
+ 'verbose': {
+ 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
+ 'datefmt': "%d/%b/%Y %H:%M:%S",
+ 'style': '{',
+ },
+ 'api': {
+ 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
+ 'style': '{'
+ }
+ },
+ 'handlers': {
+ 'mail_admins': {
+ 'level': 'ERROR',
+ 'filters': ['require_debug_false'],
+ 'class': 'django.utils.log.AdminEmailHandler'
+ },
+ 'console': {
+ 'level': 'DEBUG',
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'datetime',
+ },
+ 'file_django': {
+ 'level': 'INFO',
+ 'class': 'logging.handlers.TimedRotatingFileHandler',
+ 'filename': BASE_DIR / 'logs/django.log',
+ 'when': 'D', # interval type
+ 'interval': 1, # defaults to 1
+ 'backupCount': 10, # how many files to keep
+ 'formatter': 'verbose',
+ },
+ 'file_api': {
+ 'level': 'INFO',
+ 'class': 'logging.handlers.TimedRotatingFileHandler',
+ 'filename': BASE_DIR / 'logs/api.log',
+ 'when': 'D',
+ 'interval': 1,
+ 'backupCount': 10,
+ 'formatter': 'verbose',
+ },
+ 'file_toaster': {
+ 'level': 'INFO',
+ 'class': 'logging.handlers.TimedRotatingFileHandler',
+ 'filename': BASE_DIR / 'logs/toaster.log',
+ 'when': 'D',
+ 'interval': 1,
+ 'backupCount': 10,
+ 'formatter': 'verbose',
+ },
+ },
+ 'loggers': {
+ 'django.request': {
+ 'handlers': ['file_django', 'console'],
+ 'level': 'WARN',
+ 'propagate': True,
+ },
+ 'django': {
+ 'handlers': ['file_django', 'console'],
+ 'level': 'WARNING',
+ 'propogate': True,
+ },
+ 'toaster': {
+ 'handlers': ['file_toaster'],
+ 'level': 'INFO',
+ 'propagate': False,
+ },
+ 'api': {
+ 'handlers': ['file_api'],
+ 'level': 'INFO',
+ 'propagate': False,
+ }
+ }
+}
diff --git a/lib/toaster/toastermain/settings.py b/lib/toaster/toastermain/settings.py
index 609c85d9..b083cf58 100644
--- a/lib/toaster/toastermain/settings.py
+++ b/lib/toaster/toastermain/settings.py
@@ -9,6 +9,8 @@
# Django settings for Toaster project.
import os
+from pathlib import Path
+from toastermain.logs import LOGGING_SETTINGS
DEBUG = True
@@ -186,7 +188,13 @@ TEMPLATES = [
'django.template.loaders.app_directories.Loader',
#'django.template.loaders.eggs.Loader',
],
- 'string_if_invalid': InvalidString("%s"),
+ # [ https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled | https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled ]
+ # Generally, string_if_invalid should only be enabled in order to debug
+ # a specific template problem, then cleared once debugging is complete.
+ # If you assign a value other than '' to string_if_invalid,
+ # you will experience rendering problems with these templates and sites.
+ # 'string_if_invalid': InvalidString("%s"),
+ 'string_if_invalid': "",
'debug': DEBUG,
},
},
@@ -242,6 +250,9 @@ INSTALLED_APPS = (
'django.contrib.humanize',
'bldcollector',
'toastermain',
+
+ # 3rd-lib
+ "log_viewer",
)
@@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)):
# the site admins on every HTTP 500 error when DEBUG=False.
# See [ http://docs.djangoproject.com/en/dev/topics/logging | http://docs.djangoproject.com/en/dev/topics/logging ] for
# more details on how to customize your logging configuration.
-LOGGING = {
- 'version': 1,
- 'disable_existing_loggers': False,
- 'filters': {
- 'require_debug_false': {
- '()': 'django.utils.log.RequireDebugFalse'
- }
- },
- 'formatters': {
- 'datetime': {
- 'format': '%(asctime)s %(levelname)s %(message)s'
- }
- },
- 'handlers': {
- 'mail_admins': {
- 'level': 'ERROR',
- 'filters': ['require_debug_false'],
- 'class': 'django.utils.log.AdminEmailHandler'
- },
- 'console': {
- 'level': 'DEBUG',
- 'class': 'logging.StreamHandler',
- 'formatter': 'datetime',
- }
- },
- 'loggers': {
- 'toaster' : {
- 'handlers': ['console'],
- 'level': 'DEBUG',
- },
- 'django.request': {
- 'handlers': ['console'],
- 'level': 'WARN',
- 'propagate': True,
- },
- }
-}
+LOGGING = LOGGING_SETTINGS
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
+
+# LOG VIEWER
+# [ https://pypi.org/project/django-log-viewer/ | https://pypi.org/project/django-log-viewer/ ]
+LOG_VIEWER_FILES_PATTERN = '*.log*'
+LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs')
+LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page
+LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read
+LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
+
+# Optionally you can set the next variables in order to customize the admin:
+LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
+
if DEBUG and SQL_DEBUG:
LOGGING['loggers']['django.db.backends'] = {
diff --git a/lib/toaster/toastermain/urls.py b/lib/toaster/toastermain/urls.py
index 03603026..3be46fcf 100644
--- a/lib/toaster/toastermain/urls.py
+++ b/lib/toaster/toastermain/urls.py
@@ -28,6 +28,8 @@ urlpatterns = [
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+ url(r'^logs/', include('log_viewer.urls')),
+
# This is here to maintain backward compatibility and will be deprecated
# in the future.
url(r'^orm/eventfile$', bldcollector.views.eventfile),
Let me take a moment to say how great it is to have other folks looking at Toaster and to be able to discuss these issues with folks that understand Django. On Thu, Oct 19, 2023 at 9:51 AM Alassane Yattara < alassane.yattara@savoirfairelinux.com> wrote: > Tim, > > Seem OpenEmbedded user has not permission to log into > bitbake/lib/toaster/logs, > This is probably not where we want the logs anyway. > Yes, I can use BUILDDIR instead of BASE_DIR as logging directory, but > there is no way to know if that work in container. > > This is where we should put the logs most likely. It should be writable by the container user. > I'm trying to reproduce the issue, i cloned and run repos > https://github.com/crops/toaster-container master, > this pull crops/toaster:master, but I don't see logging system changes > there. > > The master branch container is failing, so it did not push a new container image. > Question: > - Is a way to pull a specific docker crops/toaster image that contain > logging system changes ? > > I pushed a branch with the exact scripts I use locally to test Toaster with the toaster-container. https://github.com/moto-timo/toaster-container/tree/scripts You will need to run build-and-test.sh to have the absolute latest changes in bitbake "master". It should stall at "Starting toaster...", so you will need to run "docker logs toasterserver-<UUID>" or whatever the container name is (docker ps -a will tell you). You should be able to point your web browser at http://localhost:8000 and see that toaster is in fact running and has built "quilt-native"... but the traceback is throwing of the bitbake/bin/toaster script's check for whether toaster is running or not. FWIW, you can make changes in the build-and-test.sh script to test a different branch... I've done this in the past when testing fixes. And examples of that usage are commented out in the script: #GITREPO="git:yoctoproject.org/poky-contrib" #BRANCH="timo/hardknott/toaster-fixes" Alassane > > ----- Mail original ----- > De: "Tim Orling" <ticotimo@gmail.com> > À: "Alassane Yattara" <alassane.yattara@savoirfairelinux.com> > Cc: bitbake-devel@lists.openembedded.org, toaster@lists.yoctoproject.org > Envoyé: Jeudi 19 Octobre 2023 15:00:35 > Objet: Re: [Toaster] [bitbake-devel] [PATCH] toaster: Monitoring - > implement Django logging system > > Unfortunately, this commit has broken running in a container. > > > The system will start. > Traceback (most recent call last): > File "/usr/lib/python3.8/logging/config.py", line 563, in configure > handler = self.configure_handler(handlers[name]) > File "/usr/lib/python3.8/logging/config.py", line 744, in > configure_handler > result = factory(**kwargs) > File "/usr/lib/python3.8/logging/handlers.py", line 200, in __init__ > BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) > File "/usr/lib/python3.8/logging/handlers.py", line 55, in __init__ > logging.FileHandler.__init__(self, filename, mode, encoding, delay) > File "/usr/lib/python3.8/logging/__init__.py", line 1147, in __init__ > StreamHandler.__init__(self, self._open()) > File "/usr/lib/python3.8/logging/__init__.py", line 1176, in _open > return open(self.baseFilename, self.mode, encoding=self.encoding) > PermissionError: [Errno 13] Permission denied: > '/home/usersetup/poky/bitbake/lib/toaster/logs/api.log' > > The above exception was the direct cause of the following exception: > > Traceback (most recent call last): > File "/home/usersetup/poky/bitbake/bin/../lib/toaster/manage.py", line 16, > in <module> > execute_from_command_line(sys.argv) > File > "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", > line 442, in execute_from_command_line > utility.execute() > File > "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", > line 416, in execute > django.setup() > File "/opt/venv/lib/python3.8/site-packages/django/__init__.py", line 19, > in setup > configure_logging(settings.LOGGING_CONFIG, settings.LOGGING) > File "/opt/venv/lib/python3.8/site-packages/django/utils/log.py", line 76, > in configure_logging > logging_config_func(logging_settings) > File "/usr/lib/python3.8/logging/config.py", line 808, in dictConfig > dictConfigClass(config).configure() > File "/usr/lib/python3.8/logging/config.py", line 570, in configure > raise ValueError('Unable to configure handler ' > ValueError: Unable to configure handler 'file_api' > > The reason is that BASE_DIR is resolving to bitbake/lib/toaster and the > logs are in this case are attempting to write to a read only file system. > > When running locally, this works, but perhaps is not where OpenEmbedded > users expect the logs to be: > $ ls bitbake/lib/toaster/logs > api.log toaster.log.2023-10-13 toaster.log.2023-10-16 > django.log toaster.log.2023-10-14 toaster.log.2023-10-17 > toaster.log toaster.log.2023-10-15 > > Previously, the logs were written into the build directory, like the > toaster_ui.log still is: > build-toaster-2/toaster_ui.log > > This also pointed out an issue with the toaster script: > [ https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 | > https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 ] > When we have a Python trace back, the code is not catching that there was > a failure to fully start nor fail. > e.g. "Successful start." was not output, but neither was "Toaster build > server not started." > > The health check in [ > https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 > | > https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 > ] never sees "Successful start." so unfortunately, the test stage just > eventually times out. > > On Wed, Oct 11, 2023 at 9:35 PM Tim Orling via [ > http://lists.openembedded.org/ | lists.openembedded.org ] <ticotimo= [ > mailto:gmail.com@lists.openembedded.org | gmail.com@lists.openembedded.org > ] > wrote: > > > > i think this introduces a missing dependency in toaster-requirements.txt > on django-log-viewer or similar, as there is now a failure on [ > https://github.com/crops/toaster-container | > https://github.com/crops/toaster-container ] for "master" > 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 1014, in > _gcd_import > 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 991, in > _find_and_load > 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 973, in > _find_and_load_unlocked > 03:55:49 E: 0.862 ModuleNotFoundError: No module named 'log_viewer'" > " > > > On Wed, Oct 4, 2023 at 6:45 AM Alassane Yattara < [ mailto: > alassane.yattara@savoirfairelinux.com | > alassane.yattara@savoirfairelinux.com ] > wrote: > > > --- > lib/toaster/bldcollector/views.py | 3 + > lib/toaster/logs/.gitignore | 1 + > lib/toaster/toastergui/views.py | 7 ++ > lib/toaster/toastergui/widgets.py | 4 + > lib/toaster/toastermain/logs.py | 153 ++++++++++++++++++++++++++++ > lib/toaster/toastermain/settings.py | 66 +++++------- > lib/toaster/toastermain/urls.py | 2 + > 7 files changed, 198 insertions(+), 38 deletions(-) > create mode 100644 lib/toaster/logs/.gitignore > create mode 100644 lib/toaster/toastermain/logs.py > > diff --git a/lib/toaster/bldcollector/views.py > b/lib/toaster/bldcollector/views.py > index 04cd8b3d..bdf38ae6 100644 > --- a/lib/toaster/bldcollector/views.py > +++ b/lib/toaster/bldcollector/views.py > @@ -14,8 +14,11 @@ import subprocess > import toastermain > from django.views.decorators.csrf import csrf_exempt > > +from toastermain.logs import log_view_mixin > + > > @csrf_exempt > +@log_view_mixin > def eventfile(request): > """ Receives a file by POST, and runs toaster-eventreply on this file """ > if request.method != "POST": > diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore > new file mode 100644 > index 00000000..e5ebf25a > --- /dev/null > +++ b/lib/toaster/logs/.gitignore > @@ -0,0 +1 @@ > +*.log* > diff --git a/lib/toaster/toastergui/views.py > b/lib/toaster/toastergui/views.py > index 552ff164..cc8517ba 100644 > --- a/lib/toaster/toastergui/views.py > +++ b/lib/toaster/toastergui/views.py > @@ -34,6 +34,8 @@ import mimetypes > > import logging > > +from toastermain.logs import log_view_mixin > + > logger = logging.getLogger("toaster") > > # Project creation and managed build enable > @@ -56,6 +58,7 @@ class MimeTypeFinder(object): > return guessed_type > > # single point to add global values into the context before rendering > +@log_view_mixin > def toaster_render(request, page, context): > context['project_enable'] = project_enable > context['project_specific'] = is_project_specific > @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): > return response > > from django.http import HttpResponse > +@log_view_mixin > def xhr_dirinfo(request, build_id, target_id): > top = request.GET.get('start', '/') > return HttpResponse(_get_dir_entries(build_id, target_id, top), > content_type = "application/json") > @@ -1612,6 +1616,7 @@ if True: > > from django.views.decorators.csrf import csrf_exempt > @csrf_exempt > + @log_view_mixin > def xhr_testreleasechange(request, pid): > def response(data): > return HttpResponse(jsonfilter(data), > @@ -1648,6 +1653,7 @@ if True: > except Exception as e: > return response({"error": str(e) }) > > + @log_view_mixin > def xhr_configvaredit(request, pid): > try: > prj = Project.objects.get(id = pid) > @@ -1726,6 +1732,7 @@ if True: > return HttpResponse(json.dumps({"error":str(e) + "\n" + > traceback.format_exc()}), content_type = "application/json") > > > + @log_view_mixin > def customrecipe_download(request, pid, recipe_id): > recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) > > diff --git a/lib/toaster/toastergui/widgets.py > b/lib/toaster/toastergui/widgets.py > index 53696912..51ed153a 100644 > --- a/lib/toaster/toastergui/widgets.py > +++ b/lib/toaster/toastergui/widgets.py > @@ -32,6 +32,7 @@ import re > import os > > from toastergui.tablefilter import TableFilterMap > +from toastermain.logs import log_view_mixin > > try: > from urllib import unquote_plus > @@ -84,6 +85,7 @@ class ToasterTable(TemplateView): > > return context > > + @log_view_mixin > def get(self, request, *args, **kwargs): > if request.GET.get('format', None) == 'json': > > @@ -414,6 +416,7 @@ class ToasterTypeAhead(View): > def __init__(self, *args, **kwargs): > super(ToasterTypeAhead, self).__init__() > > + @log_view_mixin > def get(self, request, *args, **kwargs): > def response(data): > return HttpResponse(json.dumps(data, > @@ -469,6 +472,7 @@ class MostRecentBuildsView(View): > > return False > > + @log_view_mixin > def get(self, request, *args, **kwargs): > """ > Returns a list of builds in JSON format. > diff --git a/lib/toaster/toastermain/logs.py > b/lib/toaster/toastermain/logs.py > new file mode 100644 > index 00000000..f9953982 > --- /dev/null > +++ b/lib/toaster/toastermain/logs.py > @@ -0,0 +1,153 @@ > +#!/usr/bin/env python3 > +# -*- coding: utf-8 -*- > + > +import logging > +import json > +from pathlib import Path > +from django.http import HttpRequest > + > +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > + > + > +def log_api_request(request, response, view, logger_name='api'): > + """Helper function for LogAPIMixin""" > + > + repjson = { > + 'view': view, > + 'path': request.path, > + 'method': request.method, > + 'status': response.status_code > + } > + > + logger = logging.getLogger(logger_name) > + [ http://logger.info/ | logger.info ] ( > + json.dumps(repjson, indent=4, separators=(", ", " : ")) > + ) > + > + > +def log_view_mixin(view): > + def log_view_request(*args, **kwargs): > + # get request from args else kwargs > + request = None > + if len(args) > 0: > + for req in args: > + if isinstance(req, HttpRequest): > + request = req > + break > + elif request is None: > + request = kwargs.get('request') > + > + response = view(*args, **kwargs) > + log_api_request( > + request, response, request.resolver_match.view_name, 'toaster') > + return response > + return log_view_request > + > + > + > +class LogAPIMixin: > + """Logs API requests > + > + tested with: > + - APIView > + - ModelViewSet > + - ReadOnlyModelViewSet > + - GenericAPIView > + > + Note: you can set `view_name` attribute in View to override > get_view_name() > + """ > + > + def get_view_name(self): > + if hasattr(self, 'view_name'): > + return self.view_name > + return super().get_view_name() > + > + def finalize_response(self, request, response, *args, **kwargs): > + log_api_request(request, response, self.get_view_name()) > + return super().finalize_response(request, response, *args, **kwargs) > + > + > +LOGGING_SETTINGS = { > + 'version': 1, > + 'disable_existing_loggers': False, > + 'filters': { > + 'require_debug_false': { > + '()': 'django.utils.log.RequireDebugFalse' > + } > + }, > + 'formatters': { > + 'datetime': { > + 'format': '%(asctime)s %(levelname)s %(message)s' > + }, > + 'verbose': { > + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} > {thread:d} {message}', > + 'datefmt': "%d/%b/%Y %H:%M:%S", > + 'style': '{', > + }, > + 'api': { > + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', > + 'style': '{' > + } > + }, > + 'handlers': { > + 'mail_admins': { > + 'level': 'ERROR', > + 'filters': ['require_debug_false'], > + 'class': 'django.utils.log.AdminEmailHandler' > + }, > + 'console': { > + 'level': 'DEBUG', > + 'class': 'logging.StreamHandler', > + 'formatter': 'datetime', > + }, > + 'file_django': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/django.log', > + 'when': 'D', # interval type > + 'interval': 1, # defaults to 1 > + 'backupCount': 10, # how many files to keep > + 'formatter': 'verbose', > + }, > + 'file_api': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/api.log', > + 'when': 'D', > + 'interval': 1, > + 'backupCount': 10, > + 'formatter': 'verbose', > + }, > + 'file_toaster': { > + 'level': 'INFO', > + 'class': 'logging.handlers.TimedRotatingFileHandler', > + 'filename': BASE_DIR / 'logs/toaster.log', > + 'when': 'D', > + 'interval': 1, > + 'backupCount': 10, > + 'formatter': 'verbose', > + }, > + }, > + 'loggers': { > + 'django.request': { > + 'handlers': ['file_django', 'console'], > + 'level': 'WARN', > + 'propagate': True, > + }, > + 'django': { > + 'handlers': ['file_django', 'console'], > + 'level': 'WARNING', > + 'propogate': True, > + }, > + 'toaster': { > + 'handlers': ['file_toaster'], > + 'level': 'INFO', > + 'propagate': False, > + }, > + 'api': { > + 'handlers': ['file_api'], > + 'level': 'INFO', > + 'propagate': False, > + } > + } > +} > diff --git a/lib/toaster/toastermain/settings.py > b/lib/toaster/toastermain/settings.py > index 609c85d9..b083cf58 100644 > --- a/lib/toaster/toastermain/settings.py > +++ b/lib/toaster/toastermain/settings.py > @@ -9,6 +9,8 @@ > # Django settings for Toaster project. > > import os > +from pathlib import Path > +from toastermain.logs import LOGGING_SETTINGS > > DEBUG = True > > @@ -186,7 +188,13 @@ TEMPLATES = [ > 'django.template.loaders.app_directories.Loader', > #'django.template.loaders.eggs.Loader', > ], > - 'string_if_invalid': InvalidString("%s"), > + # [ > https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled > | > https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled > ] > + # Generally, string_if_invalid should only be enabled in order to debug > + # a specific template problem, then cleared once debugging is complete. > + # If you assign a value other than '' to string_if_invalid, > + # you will experience rendering problems with these templates and sites. > + # 'string_if_invalid': InvalidString("%s"), > + 'string_if_invalid': "", > 'debug': DEBUG, > }, > }, > @@ -242,6 +250,9 @@ INSTALLED_APPS = ( > 'django.contrib.humanize', > 'bldcollector', > 'toastermain', > + > + # 3rd-lib > + "log_viewer", > ) > > > @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): > # the site admins on every HTTP 500 error when DEBUG=False. > # See [ http://docs.djangoproject.com/en/dev/topics/logging | > http://docs.djangoproject.com/en/dev/topics/logging ] for > # more details on how to customize your logging configuration. > -LOGGING = { > - 'version': 1, > - 'disable_existing_loggers': False, > - 'filters': { > - 'require_debug_false': { > - '()': 'django.utils.log.RequireDebugFalse' > - } > - }, > - 'formatters': { > - 'datetime': { > - 'format': '%(asctime)s %(levelname)s %(message)s' > - } > - }, > - 'handlers': { > - 'mail_admins': { > - 'level': 'ERROR', > - 'filters': ['require_debug_false'], > - 'class': 'django.utils.log.AdminEmailHandler' > - }, > - 'console': { > - 'level': 'DEBUG', > - 'class': 'logging.StreamHandler', > - 'formatter': 'datetime', > - } > - }, > - 'loggers': { > - 'toaster' : { > - 'handlers': ['console'], > - 'level': 'DEBUG', > - }, > - 'django.request': { > - 'handlers': ['console'], > - 'level': 'WARN', > - 'propagate': True, > - }, > - } > -} > +LOGGING = LOGGING_SETTINGS > + > +# Build paths inside the project like this: BASE_DIR / 'subdir'. > +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > + > +# LOG VIEWER > +# [ https://pypi.org/project/django-log-viewer/ | > https://pypi.org/project/django-log-viewer/ ] > +LOG_VIEWER_FILES_PATTERN = '*.log*' > +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') > +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page > +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read > +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] > + > +# Optionally you can set the next variables in order to customize the > admin: > +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" > + > > if DEBUG and SQL_DEBUG: > LOGGING['loggers']['django.db.backends'] = { > diff --git a/lib/toaster/toastermain/urls.py > b/lib/toaster/toastermain/urls.py > index 03603026..3be46fcf 100644 > --- a/lib/toaster/toastermain/urls.py > +++ b/lib/toaster/toastermain/urls.py > @@ -28,6 +28,8 @@ urlpatterns = [ > # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), > > > + url(r'^logs/', include('log_viewer.urls')), > + > # This is here to maintain backward compatibility and will be deprecated > # in the future. > url(r'^orm/eventfile$', bldcollector.views.eventfile), > -- > 2.34.1 > > > > > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#5881): > https://lists.yoctoproject.org/g/toaster/message/5881 > Mute This Topic: https://lists.yoctoproject.org/mt/102060497/7896845 > Group Owner: toaster+owner@lists.yoctoproject.org > Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [ > alassane.yattara@savoirfairelinux.com] > -=-=-=-=-=-=-=-=-=-=-=- >
On Thu, Oct 19, 2023 at 10:12 AM Tim Orling via lists.yoctoproject.org <ticotimo=gmail.com@lists.yoctoproject.org> wrote: > Let me take a moment to say how great it is to have other folks looking at > Toaster and to be able to discuss these issues with folks that understand > Django. > > On Thu, Oct 19, 2023 at 9:51 AM Alassane Yattara < > alassane.yattara@savoirfairelinux.com> wrote: > >> Tim, >> >> Seem OpenEmbedded user has not permission to log into >> bitbake/lib/toaster/logs, >> > > This is probably not where we want the logs anyway. > > >> Yes, I can use BUILDDIR instead of BASE_DIR as logging directory, but >> there is no way to know if that work in container. >> >> > This is where we should put the logs most likely. It should be writable by > the container user. > > >> I'm trying to reproduce the issue, i cloned and run repos >> https://github.com/crops/toaster-container master, >> this pull crops/toaster:master, but I don't see logging system changes >> there. >> >> > The master branch container is failing, so it did not push a new container > image. > > >> Question: >> - Is a way to pull a specific docker crops/toaster image that contain >> logging system changes ? >> >> > I pushed a branch with the exact scripts I use locally to test Toaster > with the toaster-container. > https://github.com/moto-timo/toaster-container/tree/scripts > > You will need to run build-and-test.sh to have the absolute latest changes > in bitbake "master". > It should stall at "Starting toaster...", so you will need to run "docker > logs toasterserver-<UUID>" > or whatever the container name is (docker ps -a will tell you). > docker logs toasterserver-<UUID> will allow you to see the otherwise hidden Python traceback. > > You should be able to point your web browser at http://localhost:8000 and > see that toaster is in fact > running and has built "quilt-native"... but the traceback is throwing of > the bitbake/bin/toaster script's > check for whether toaster is running or not. > Correction. I had a local (non containerized) Toaster instance also running. You should NOT be able to connect with the browser. > FWIW, you can make changes in the build-and-test.sh script to test a > different branch... I've done this > in the past when testing fixes. And examples of that usage are commented > out in the script: > #GITREPO="git:yoctoproject.org/poky-contrib" > #BRANCH="timo/hardknott/toaster-fixes" > > Alassane >> >> ----- Mail original ----- >> De: "Tim Orling" <ticotimo@gmail.com> >> À: "Alassane Yattara" <alassane.yattara@savoirfairelinux.com> >> Cc: bitbake-devel@lists.openembedded.org, toaster@lists.yoctoproject.org >> Envoyé: Jeudi 19 Octobre 2023 15:00:35 >> Objet: Re: [Toaster] [bitbake-devel] [PATCH] toaster: Monitoring - >> implement Django logging system >> >> Unfortunately, this commit has broken running in a container. >> >> >> The system will start. >> Traceback (most recent call last): >> File "/usr/lib/python3.8/logging/config.py", line 563, in configure >> handler = self.configure_handler(handlers[name]) >> File "/usr/lib/python3.8/logging/config.py", line 744, in >> configure_handler >> result = factory(**kwargs) >> File "/usr/lib/python3.8/logging/handlers.py", line 200, in __init__ >> BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) >> File "/usr/lib/python3.8/logging/handlers.py", line 55, in __init__ >> logging.FileHandler.__init__(self, filename, mode, encoding, delay) >> File "/usr/lib/python3.8/logging/__init__.py", line 1147, in __init__ >> StreamHandler.__init__(self, self._open()) >> File "/usr/lib/python3.8/logging/__init__.py", line 1176, in _open >> return open(self.baseFilename, self.mode, encoding=self.encoding) >> PermissionError: [Errno 13] Permission denied: >> '/home/usersetup/poky/bitbake/lib/toaster/logs/api.log' >> >> The above exception was the direct cause of the following exception: >> >> Traceback (most recent call last): >> File "/home/usersetup/poky/bitbake/bin/../lib/toaster/manage.py", line >> 16, in <module> >> execute_from_command_line(sys.argv) >> File >> "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", >> line 442, in execute_from_command_line >> utility.execute() >> File >> "/opt/venv/lib/python3.8/site-packages/django/core/management/__init__.py", >> line 416, in execute >> django.setup() >> File "/opt/venv/lib/python3.8/site-packages/django/__init__.py", line 19, >> in setup >> configure_logging(settings.LOGGING_CONFIG, settings.LOGGING) >> File "/opt/venv/lib/python3.8/site-packages/django/utils/log.py", line >> 76, in configure_logging >> logging_config_func(logging_settings) >> File "/usr/lib/python3.8/logging/config.py", line 808, in dictConfig >> dictConfigClass(config).configure() >> File "/usr/lib/python3.8/logging/config.py", line 570, in configure >> raise ValueError('Unable to configure handler ' >> ValueError: Unable to configure handler 'file_api' >> >> The reason is that BASE_DIR is resolving to bitbake/lib/toaster and the >> logs are in this case are attempting to write to a read only file system. >> >> When running locally, this works, but perhaps is not where OpenEmbedded >> users expect the logs to be: >> $ ls bitbake/lib/toaster/logs >> api.log toaster.log.2023-10-13 toaster.log.2023-10-16 >> django.log toaster.log.2023-10-14 toaster.log.2023-10-17 >> toaster.log toaster.log.2023-10-15 >> >> Previously, the logs were written into the build directory, like the >> toaster_ui.log still is: >> build-toaster-2/toaster_ui.log >> >> This also pointed out an issue with the toaster script: >> [ https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 | >> https://git.yoctoproject.org/poky/tree/bitbake/bin/toaster#n308 ] >> When we have a Python trace back, the code is not catching that there was >> a failure to fully start nor fail. >> e.g. "Successful start." was not output, but neither was "Toaster build >> server not started." >> >> The health check in [ >> https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 >> | >> https://github.com/crops/toaster-container/blob/master/tests/runtests.sh#L91 >> ] never sees "Successful start." so unfortunately, the test stage just >> eventually times out. >> >> On Wed, Oct 11, 2023 at 9:35 PM Tim Orling via [ >> http://lists.openembedded.org/ | lists.openembedded.org ] <ticotimo= [ >> mailto:gmail.com@lists.openembedded.org | >> gmail.com@lists.openembedded.org ] > wrote: >> >> >> >> i think this introduces a missing dependency in toaster-requirements.txt >> on django-log-viewer or similar, as there is now a failure on [ >> https://github.com/crops/toaster-container | >> https://github.com/crops/toaster-container ] for "master" >> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 1014, in >> _gcd_import >> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 991, in >> _find_and_load >> 03:55:49 E: 0.862 File "<frozen importlib._bootstrap>", line 973, in >> _find_and_load_unlocked >> 03:55:49 E: 0.862 ModuleNotFoundError: No module named 'log_viewer'" >> " >> >> >> On Wed, Oct 4, 2023 at 6:45 AM Alassane Yattara < [ mailto: >> alassane.yattara@savoirfairelinux.com | >> alassane.yattara@savoirfairelinux.com ] > wrote: >> >> >> --- >> lib/toaster/bldcollector/views.py | 3 + >> lib/toaster/logs/.gitignore | 1 + >> lib/toaster/toastergui/views.py | 7 ++ >> lib/toaster/toastergui/widgets.py | 4 + >> lib/toaster/toastermain/logs.py | 153 ++++++++++++++++++++++++++++ >> lib/toaster/toastermain/settings.py | 66 +++++------- >> lib/toaster/toastermain/urls.py | 2 + >> 7 files changed, 198 insertions(+), 38 deletions(-) >> create mode 100644 lib/toaster/logs/.gitignore >> create mode 100644 lib/toaster/toastermain/logs.py >> >> diff --git a/lib/toaster/bldcollector/views.py >> b/lib/toaster/bldcollector/views.py >> index 04cd8b3d..bdf38ae6 100644 >> --- a/lib/toaster/bldcollector/views.py >> +++ b/lib/toaster/bldcollector/views.py >> @@ -14,8 +14,11 @@ import subprocess >> import toastermain >> from django.views.decorators.csrf import csrf_exempt >> >> +from toastermain.logs import log_view_mixin >> + >> >> @csrf_exempt >> +@log_view_mixin >> def eventfile(request): >> """ Receives a file by POST, and runs toaster-eventreply on this file """ >> if request.method != "POST": >> diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore >> new file mode 100644 >> index 00000000..e5ebf25a >> --- /dev/null >> +++ b/lib/toaster/logs/.gitignore >> @@ -0,0 +1 @@ >> +*.log* >> diff --git a/lib/toaster/toastergui/views.py >> b/lib/toaster/toastergui/views.py >> index 552ff164..cc8517ba 100644 >> --- a/lib/toaster/toastergui/views.py >> +++ b/lib/toaster/toastergui/views.py >> @@ -34,6 +34,8 @@ import mimetypes >> >> import logging >> >> +from toastermain.logs import log_view_mixin >> + >> logger = logging.getLogger("toaster") >> >> # Project creation and managed build enable >> @@ -56,6 +58,7 @@ class MimeTypeFinder(object): >> return guessed_type >> >> # single point to add global values into the context before rendering >> +@log_view_mixin >> def toaster_render(request, page, context): >> context['project_enable'] = project_enable >> context['project_specific'] = is_project_specific >> @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): >> return response >> >> from django.http import HttpResponse >> +@log_view_mixin >> def xhr_dirinfo(request, build_id, target_id): >> top = request.GET.get('start', '/') >> return HttpResponse(_get_dir_entries(build_id, target_id, top), >> content_type = "application/json") >> @@ -1612,6 +1616,7 @@ if True: >> >> from django.views.decorators.csrf import csrf_exempt >> @csrf_exempt >> + @log_view_mixin >> def xhr_testreleasechange(request, pid): >> def response(data): >> return HttpResponse(jsonfilter(data), >> @@ -1648,6 +1653,7 @@ if True: >> except Exception as e: >> return response({"error": str(e) }) >> >> + @log_view_mixin >> def xhr_configvaredit(request, pid): >> try: >> prj = Project.objects.get(id = pid) >> @@ -1726,6 +1732,7 @@ if True: >> return HttpResponse(json.dumps({"error":str(e) + "\n" + >> traceback.format_exc()}), content_type = "application/json") >> >> >> + @log_view_mixin >> def customrecipe_download(request, pid, recipe_id): >> recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) >> >> diff --git a/lib/toaster/toastergui/widgets.py >> b/lib/toaster/toastergui/widgets.py >> index 53696912..51ed153a 100644 >> --- a/lib/toaster/toastergui/widgets.py >> +++ b/lib/toaster/toastergui/widgets.py >> @@ -32,6 +32,7 @@ import re >> import os >> >> from toastergui.tablefilter import TableFilterMap >> +from toastermain.logs import log_view_mixin >> >> try: >> from urllib import unquote_plus >> @@ -84,6 +85,7 @@ class ToasterTable(TemplateView): >> >> return context >> >> + @log_view_mixin >> def get(self, request, *args, **kwargs): >> if request.GET.get('format', None) == 'json': >> >> @@ -414,6 +416,7 @@ class ToasterTypeAhead(View): >> def __init__(self, *args, **kwargs): >> super(ToasterTypeAhead, self).__init__() >> >> + @log_view_mixin >> def get(self, request, *args, **kwargs): >> def response(data): >> return HttpResponse(json.dumps(data, >> @@ -469,6 +472,7 @@ class MostRecentBuildsView(View): >> >> return False >> >> + @log_view_mixin >> def get(self, request, *args, **kwargs): >> """ >> Returns a list of builds in JSON format. >> diff --git a/lib/toaster/toastermain/logs.py >> b/lib/toaster/toastermain/logs.py >> new file mode 100644 >> index 00000000..f9953982 >> --- /dev/null >> +++ b/lib/toaster/toastermain/logs.py >> @@ -0,0 +1,153 @@ >> +#!/usr/bin/env python3 >> +# -*- coding: utf-8 -*- >> + >> +import logging >> +import json >> +from pathlib import Path >> +from django.http import HttpRequest >> + >> +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent >> + >> + >> +def log_api_request(request, response, view, logger_name='api'): >> + """Helper function for LogAPIMixin""" >> + >> + repjson = { >> + 'view': view, >> + 'path': request.path, >> + 'method': request.method, >> + 'status': response.status_code >> + } >> + >> + logger = logging.getLogger(logger_name) >> + [ http://logger.info/ | logger.info ] ( >> + json.dumps(repjson, indent=4, separators=(", ", " : ")) >> + ) >> + >> + >> +def log_view_mixin(view): >> + def log_view_request(*args, **kwargs): >> + # get request from args else kwargs >> + request = None >> + if len(args) > 0: >> + for req in args: >> + if isinstance(req, HttpRequest): >> + request = req >> + break >> + elif request is None: >> + request = kwargs.get('request') >> + >> + response = view(*args, **kwargs) >> + log_api_request( >> + request, response, request.resolver_match.view_name, 'toaster') >> + return response >> + return log_view_request >> + >> + >> + >> +class LogAPIMixin: >> + """Logs API requests >> + >> + tested with: >> + - APIView >> + - ModelViewSet >> + - ReadOnlyModelViewSet >> + - GenericAPIView >> + >> + Note: you can set `view_name` attribute in View to override >> get_view_name() >> + """ >> + >> + def get_view_name(self): >> + if hasattr(self, 'view_name'): >> + return self.view_name >> + return super().get_view_name() >> + >> + def finalize_response(self, request, response, *args, **kwargs): >> + log_api_request(request, response, self.get_view_name()) >> + return super().finalize_response(request, response, *args, **kwargs) >> + >> + >> +LOGGING_SETTINGS = { >> + 'version': 1, >> + 'disable_existing_loggers': False, >> + 'filters': { >> + 'require_debug_false': { >> + '()': 'django.utils.log.RequireDebugFalse' >> + } >> + }, >> + 'formatters': { >> + 'datetime': { >> + 'format': '%(asctime)s %(levelname)s %(message)s' >> + }, >> + 'verbose': { >> + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} >> {thread:d} {message}', >> + 'datefmt': "%d/%b/%Y %H:%M:%S", >> + 'style': '{', >> + }, >> + 'api': { >> + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', >> + 'style': '{' >> + } >> + }, >> + 'handlers': { >> + 'mail_admins': { >> + 'level': 'ERROR', >> + 'filters': ['require_debug_false'], >> + 'class': 'django.utils.log.AdminEmailHandler' >> + }, >> + 'console': { >> + 'level': 'DEBUG', >> + 'class': 'logging.StreamHandler', >> + 'formatter': 'datetime', >> + }, >> + 'file_django': { >> + 'level': 'INFO', >> + 'class': 'logging.handlers.TimedRotatingFileHandler', >> + 'filename': BASE_DIR / 'logs/django.log', >> + 'when': 'D', # interval type >> + 'interval': 1, # defaults to 1 >> + 'backupCount': 10, # how many files to keep >> + 'formatter': 'verbose', >> + }, >> + 'file_api': { >> + 'level': 'INFO', >> + 'class': 'logging.handlers.TimedRotatingFileHandler', >> + 'filename': BASE_DIR / 'logs/api.log', >> + 'when': 'D', >> + 'interval': 1, >> + 'backupCount': 10, >> + 'formatter': 'verbose', >> + }, >> + 'file_toaster': { >> + 'level': 'INFO', >> + 'class': 'logging.handlers.TimedRotatingFileHandler', >> + 'filename': BASE_DIR / 'logs/toaster.log', >> + 'when': 'D', >> + 'interval': 1, >> + 'backupCount': 10, >> + 'formatter': 'verbose', >> + }, >> + }, >> + 'loggers': { >> + 'django.request': { >> + 'handlers': ['file_django', 'console'], >> + 'level': 'WARN', >> + 'propagate': True, >> + }, >> + 'django': { >> + 'handlers': ['file_django', 'console'], >> + 'level': 'WARNING', >> + 'propogate': True, >> + }, >> + 'toaster': { >> + 'handlers': ['file_toaster'], >> + 'level': 'INFO', >> + 'propagate': False, >> + }, >> + 'api': { >> + 'handlers': ['file_api'], >> + 'level': 'INFO', >> + 'propagate': False, >> + } >> + } >> +} >> diff --git a/lib/toaster/toastermain/settings.py >> b/lib/toaster/toastermain/settings.py >> index 609c85d9..b083cf58 100644 >> --- a/lib/toaster/toastermain/settings.py >> +++ b/lib/toaster/toastermain/settings.py >> @@ -9,6 +9,8 @@ >> # Django settings for Toaster project. >> >> import os >> +from pathlib import Path >> +from toastermain.logs import LOGGING_SETTINGS >> >> DEBUG = True >> >> @@ -186,7 +188,13 @@ TEMPLATES = [ >> 'django.template.loaders.app_directories.Loader', >> #'django.template.loaders.eggs.Loader', >> ], >> - 'string_if_invalid': InvalidString("%s"), >> + # [ >> https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled >> | >> https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled >> ] >> + # Generally, string_if_invalid should only be enabled in order to debug >> + # a specific template problem, then cleared once debugging is complete. >> + # If you assign a value other than '' to string_if_invalid, >> + # you will experience rendering problems with these templates and >> sites. >> + # 'string_if_invalid': InvalidString("%s"), >> + 'string_if_invalid': "", >> 'debug': DEBUG, >> }, >> }, >> @@ -242,6 +250,9 @@ INSTALLED_APPS = ( >> 'django.contrib.humanize', >> 'bldcollector', >> 'toastermain', >> + >> + # 3rd-lib >> + "log_viewer", >> ) >> >> >> @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): >> # the site admins on every HTTP 500 error when DEBUG=False. >> # See [ http://docs.djangoproject.com/en/dev/topics/logging | >> http://docs.djangoproject.com/en/dev/topics/logging ] for >> # more details on how to customize your logging configuration. >> -LOGGING = { >> - 'version': 1, >> - 'disable_existing_loggers': False, >> - 'filters': { >> - 'require_debug_false': { >> - '()': 'django.utils.log.RequireDebugFalse' >> - } >> - }, >> - 'formatters': { >> - 'datetime': { >> - 'format': '%(asctime)s %(levelname)s %(message)s' >> - } >> - }, >> - 'handlers': { >> - 'mail_admins': { >> - 'level': 'ERROR', >> - 'filters': ['require_debug_false'], >> - 'class': 'django.utils.log.AdminEmailHandler' >> - }, >> - 'console': { >> - 'level': 'DEBUG', >> - 'class': 'logging.StreamHandler', >> - 'formatter': 'datetime', >> - } >> - }, >> - 'loggers': { >> - 'toaster' : { >> - 'handlers': ['console'], >> - 'level': 'DEBUG', >> - }, >> - 'django.request': { >> - 'handlers': ['console'], >> - 'level': 'WARN', >> - 'propagate': True, >> - }, >> - } >> -} >> +LOGGING = LOGGING_SETTINGS >> + >> +# Build paths inside the project like this: BASE_DIR / 'subdir'. >> +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent >> + >> +# LOG VIEWER >> +# [ https://pypi.org/project/django-log-viewer/ | >> https://pypi.org/project/django-log-viewer/ ] >> +LOG_VIEWER_FILES_PATTERN = '*.log*' >> +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') >> +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page >> +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read >> +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] >> + >> +# Optionally you can set the next variables in order to customize the >> admin: >> +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" >> + >> >> if DEBUG and SQL_DEBUG: >> LOGGING['loggers']['django.db.backends'] = { >> diff --git a/lib/toaster/toastermain/urls.py >> b/lib/toaster/toastermain/urls.py >> index 03603026..3be46fcf 100644 >> --- a/lib/toaster/toastermain/urls.py >> +++ b/lib/toaster/toastermain/urls.py >> @@ -28,6 +28,8 @@ urlpatterns = [ >> # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), >> >> >> + url(r'^logs/', include('log_viewer.urls')), >> + >> # This is here to maintain backward compatibility and will be deprecated >> # in the future. >> url(r'^orm/eventfile$', bldcollector.views.eventfile), >> -- >> 2.34.1 >> >> >> >> >> >> >> >> >> >> >> > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#5884): > https://lists.yoctoproject.org/g/toaster/message/5884 > Mute This Topic: https://lists.yoctoproject.org/mt/102060497/924729 > Group Owner: toaster+owner@lists.yoctoproject.org > Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
diff --git a/lib/toaster/bldcollector/views.py b/lib/toaster/bldcollector/views.py index 04cd8b3d..bdf38ae6 100644 --- a/lib/toaster/bldcollector/views.py +++ b/lib/toaster/bldcollector/views.py @@ -14,8 +14,11 @@ import subprocess import toastermain from django.views.decorators.csrf import csrf_exempt +from toastermain.logs import log_view_mixin + @csrf_exempt +@log_view_mixin def eventfile(request): """ Receives a file by POST, and runs toaster-eventreply on this file """ if request.method != "POST": diff --git a/lib/toaster/logs/.gitignore b/lib/toaster/logs/.gitignore new file mode 100644 index 00000000..e5ebf25a --- /dev/null +++ b/lib/toaster/logs/.gitignore @@ -0,0 +1 @@ +*.log* diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 552ff164..cc8517ba 100644 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -34,6 +34,8 @@ import mimetypes import logging +from toastermain.logs import log_view_mixin + logger = logging.getLogger("toaster") # Project creation and managed build enable @@ -56,6 +58,7 @@ class MimeTypeFinder(object): return guessed_type # single point to add global values into the context before rendering +@log_view_mixin def toaster_render(request, page, context): context['project_enable'] = project_enable context['project_specific'] = is_project_specific @@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): return response from django.http import HttpResponse +@log_view_mixin def xhr_dirinfo(request, build_id, target_id): top = request.GET.get('start', '/') return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") @@ -1612,6 +1616,7 @@ if True: from django.views.decorators.csrf import csrf_exempt @csrf_exempt + @log_view_mixin def xhr_testreleasechange(request, pid): def response(data): return HttpResponse(jsonfilter(data), @@ -1648,6 +1653,7 @@ if True: except Exception as e: return response({"error": str(e) }) + @log_view_mixin def xhr_configvaredit(request, pid): try: prj = Project.objects.get(id = pid) @@ -1726,6 +1732,7 @@ if True: return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") + @log_view_mixin def customrecipe_download(request, pid, recipe_id): recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) diff --git a/lib/toaster/toastergui/widgets.py b/lib/toaster/toastergui/widgets.py index 53696912..51ed153a 100644 --- a/lib/toaster/toastergui/widgets.py +++ b/lib/toaster/toastergui/widgets.py @@ -32,6 +32,7 @@ import re import os from toastergui.tablefilter import TableFilterMap +from toastermain.logs import log_view_mixin try: from urllib import unquote_plus @@ -84,6 +85,7 @@ class ToasterTable(TemplateView): return context + @log_view_mixin def get(self, request, *args, **kwargs): if request.GET.get('format', None) == 'json': @@ -414,6 +416,7 @@ class ToasterTypeAhead(View): def __init__(self, *args, **kwargs): super(ToasterTypeAhead, self).__init__() + @log_view_mixin def get(self, request, *args, **kwargs): def response(data): return HttpResponse(json.dumps(data, @@ -469,6 +472,7 @@ class MostRecentBuildsView(View): return False + @log_view_mixin def get(self, request, *args, **kwargs): """ Returns a list of builds in JSON format. diff --git a/lib/toaster/toastermain/logs.py b/lib/toaster/toastermain/logs.py new file mode 100644 index 00000000..f9953982 --- /dev/null +++ b/lib/toaster/toastermain/logs.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging +import json +from pathlib import Path +from django.http import HttpRequest + +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent + + +def log_api_request(request, response, view, logger_name='api'): + """Helper function for LogAPIMixin""" + + repjson = { + 'view': view, + 'path': request.path, + 'method': request.method, + 'status': response.status_code + } + + logger = logging.getLogger(logger_name) + logger.info( + json.dumps(repjson, indent=4, separators=(", ", " : ")) + ) + + +def log_view_mixin(view): + def log_view_request(*args, **kwargs): + # get request from args else kwargs + request = None + if len(args) > 0: + for req in args: + if isinstance(req, HttpRequest): + request = req + break + elif request is None: + request = kwargs.get('request') + + response = view(*args, **kwargs) + log_api_request( + request, response, request.resolver_match.view_name, 'toaster') + return response + return log_view_request + + + +class LogAPIMixin: + """Logs API requests + + tested with: + - APIView + - ModelViewSet + - ReadOnlyModelViewSet + - GenericAPIView + + Note: you can set `view_name` attribute in View to override get_view_name() + """ + + def get_view_name(self): + if hasattr(self, 'view_name'): + return self.view_name + return super().get_view_name() + + def finalize_response(self, request, response, *args, **kwargs): + log_api_request(request, response, self.get_view_name()) + return super().finalize_response(request, response, *args, **kwargs) + + +LOGGING_SETTINGS = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'datetime': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'verbose': { + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', + 'datefmt': "%d/%b/%Y %H:%M:%S", + 'style': '{', + }, + 'api': { + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', + 'style': '{' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'datetime', + }, + 'file_django': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/django.log', + 'when': 'D', # interval type + 'interval': 1, # defaults to 1 + 'backupCount': 10, # how many files to keep + 'formatter': 'verbose', + }, + 'file_api': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/api.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + 'file_toaster': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BASE_DIR / 'logs/toaster.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['file_django', 'console'], + 'level': 'WARN', + 'propagate': True, + }, + 'django': { + 'handlers': ['file_django', 'console'], + 'level': 'WARNING', + 'propogate': True, + }, + 'toaster': { + 'handlers': ['file_toaster'], + 'level': 'INFO', + 'propagate': False, + }, + 'api': { + 'handlers': ['file_api'], + 'level': 'INFO', + 'propagate': False, + } + } +} diff --git a/lib/toaster/toastermain/settings.py b/lib/toaster/toastermain/settings.py index 609c85d9..b083cf58 100644 --- a/lib/toaster/toastermain/settings.py +++ b/lib/toaster/toastermain/settings.py @@ -9,6 +9,8 @@ # Django settings for Toaster project. import os +from pathlib import Path +from toastermain.logs import LOGGING_SETTINGS DEBUG = True @@ -186,7 +188,13 @@ TEMPLATES = [ 'django.template.loaders.app_directories.Loader', #'django.template.loaders.eggs.Loader', ], - 'string_if_invalid': InvalidString("%s"), + # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled + # Generally, string_if_invalid should only be enabled in order to debug + # a specific template problem, then cleared once debugging is complete. + # If you assign a value other than '' to string_if_invalid, + # you will experience rendering problems with these templates and sites. + # 'string_if_invalid': InvalidString("%s"), + 'string_if_invalid': "", 'debug': DEBUG, }, }, @@ -242,6 +250,9 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'bldcollector', 'toastermain', + + # 3rd-lib + "log_viewer", ) @@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): # the site admins on every HTTP 500 error when DEBUG=False. # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - } - }, - 'formatters': { - 'datetime': { - 'format': '%(asctime)s %(levelname)s %(message)s' - } - }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'datetime', - } - }, - 'loggers': { - 'toaster' : { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'WARN', - 'propagate': True, - }, - } -} +LOGGING = LOGGING_SETTINGS + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve(strict=True).parent.parent + +# LOG VIEWER +# https://pypi.org/project/django-log-viewer/ +LOG_VIEWER_FILES_PATTERN = '*.log*' +LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] + +# Optionally you can set the next variables in order to customize the admin: +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" + if DEBUG and SQL_DEBUG: LOGGING['loggers']['django.db.backends'] = { diff --git a/lib/toaster/toastermain/urls.py b/lib/toaster/toastermain/urls.py index 03603026..3be46fcf 100644 --- a/lib/toaster/toastermain/urls.py +++ b/lib/toaster/toastermain/urls.py @@ -28,6 +28,8 @@ urlpatterns = [ # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + url(r'^logs/', include('log_viewer.urls')), + # This is here to maintain backward compatibility and will be deprecated # in the future. url(r'^orm/eventfile$', bldcollector.views.eventfile),