# # SPDX-License-Identifier: GPL-2.0-only # from django.core.management.base import BaseCommand from django.db import transaction from django.db.models import Q from bldcontrol.bbcontroller import getBuildEnvironmentController from bldcontrol.models import BuildRequest, BuildEnvironment from bldcontrol.models import BRError, BRVariable from orm.models import Build, LogMessage, Target import logging import traceback import signal import os logger = logging.getLogger("toaster") class Command(BaseCommand): args = "" help = "Schedules and executes build requests as possible. "\ "Does not return (interrupt with Ctrl-C)" @transaction.atomic def _selectBuildEnvironment(self): bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) bec.be.lock = BuildEnvironment.LOCK_LOCK bec.be.save() return bec @transaction.atomic def _selectBuildRequest(self): br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() return br def schedule(self): try: # select the build environment and the request to build br = self._selectBuildRequest() if br: br.state = BuildRequest.REQ_INPROGRESS br.save() else: return try: bec = self._selectBuildEnvironment() except IndexError as e: # we could not find a BEC; postpone the BR br.state = BuildRequest.REQ_QUEUED br.save() logger.debug("runbuilds: No build env (%s)" % e) return logger.info("runbuilds: starting build %s, environment %s" % (br, bec.be)) # let the build request know where it is being executed br.environment = bec.be br.save() # this triggers an async build bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all(), "%d:%d" % (br.pk, bec.be.pk)) except Exception as e: logger.error("runbuilds: Error launching build %s" % e) traceback.print_exc() if "[Errno 111] Connection refused" in str(e): # Connection refused, read toaster_server.out errmsg = bec.readServerLogFile() else: errmsg = str(e) BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, traceback=traceback.format_exc()) br.state = BuildRequest.REQ_FAILED br.save() bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() # Cancel the pending build and report the exception to the UI log_object = LogMessage.objects.create( build = br.build, level = LogMessage.EXCEPTION, message = errmsg) log_object.save() br.build.outcome = Build.FAILED br.build.save() def archive(self): for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): if br.build is None: br.state = BuildRequest.REQ_FAILED else: br.state = BuildRequest.REQ_COMPLETED br.save() def cleanup(self): from django.utils import timezone from datetime import timedelta # environments locked for more than 30 seconds # they should be unlocked BuildEnvironment.objects.filter( Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED, BuildRequest.REQ_CANCELLING]) & Q(lock=BuildEnvironment.LOCK_LOCK) & Q(updated__lt=timezone.now() - timedelta(seconds=30)) ).update(lock=BuildEnvironment.LOCK_FREE) # update all Builds that were in progress and failed to start for br in BuildRequest.objects.filter( state=BuildRequest.REQ_FAILED, build__outcome=Build.IN_PROGRESS): # transpose the launch errors in ToasterExceptions br.build.outcome = Build.FAILED for brerror in br.brerror_set.all(): logger.debug("Saving error %s" % brerror) LogMessage.objects.create(build=br.build, level=LogMessage.EXCEPTION, message=brerror.errmsg) br.build.save() # we don't have a true build object here; hence, toasterui # didn't have a change to release the BE lock br.environment.lock = BuildEnvironment.LOCK_FREE br.environment.save() # update all BuildRequests without a build created for br in BuildRequest.objects.filter(build=None): br.build = Build.objects.create(project=br.project, completed_on=br.updated, started_on=br.created) br.build.outcome = Build.FAILED try: br.build.machine = br.brvariable_set.get(name='MACHINE').value except BRVariable.DoesNotExist: pass br.save() # transpose target information for brtarget in br.brtarget_set.all(): Target.objects.create(build=br.build, target=brtarget.target, task=brtarget.task) # transpose the launch errors in ToasterExceptions for brerror in br.brerror_set.all(): LogMessage.objects.create(build=br.build, level=LogMessage.EXCEPTION, message=brerror.errmsg) br.build.save() # Make sure the LOCK is removed for builds which have been fully # cancelled for br in BuildRequest.objects.filter( Q(build__outcome=Build.CANCELLED) & Q(state=BuildRequest.REQ_CANCELLING) & ~Q(environment=None)): br.environment.lock = BuildEnvironment.LOCK_FREE br.environment.save() def runbuild(self): try: self.cleanup() except Exception as e: logger.warning("runbuilds: cleanup exception %s" % str(e)) try: self.archive() except Exception as e: logger.warning("runbuilds: archive exception %s" % str(e)) try: self.schedule() except Exception as e: logger.warning("runbuilds: schedule exception %s" % str(e)) # Test to see if a build pre-maturely died due to a bitbake crash def check_dead_builds(self): do_cleanup = False try: for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): # Get the build directory if br.project.builddir: builddir = br.project.builddir else: builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id) # Check log to see if there is a recent traceback toaster_ui_log = os.path.join(builddir, 'toaster_ui.log') test_file = os.path.join(builddir, '._toaster_check.txt') os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file)) traceback_text = '' is_traceback = False with open(test_file,'r') as test_file_fd: test_file_tail = test_file_fd.readlines() for line in test_file_tail: if line.startswith('Traceback (most recent call last):'): traceback_text = line is_traceback = True elif line.startswith('NOTE: ToasterUI waiting for events'): # Ignore any traceback before new build start traceback_text = '' is_traceback = False elif line.startswith('Note: Toaster traceback auto-stop'): # Ignore any traceback before this previous traceback catch traceback_text = '' is_traceback = False elif is_traceback: traceback_text += line # Test the results is_stop = False if is_traceback: # Found a traceback errtype = 'Bitbake crash' errmsg = 'Bitbake crash\n' + traceback_text state = BuildRequest.REQ_FAILED # Clean up bitbake files bitbake_lock = os.path.join(builddir, 'bitbake.lock') if os.path.isfile(bitbake_lock): os.remove(bitbake_lock) bitbake_sock = os.path.join(builddir, 'bitbake.sock') if os.path.isfile(bitbake_sock): os.remove(bitbake_sock) if os.path.isfile(test_file): os.remove(test_file) # Add note to ignore this traceback on next check os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log) is_stop = True # Add more tests here #elif ... # Stop the build request? if is_stop: brerror = BRError( req = br, errtype = errtype, errmsg = errmsg, traceback = traceback_text, ) brerror.save() br.state = state br.save() do_cleanup = True # Do cleanup if do_cleanup: self.cleanup() except Exception as e: logger.error("runbuilds: Error in check_dead_builds %s" % e) def handle(self, **options): pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), ".runbuilds.pid") with open(pidfile_path, 'w') as pidfile: pidfile.write("%s" % os.getpid()) # Clean up any stale/failed builds from previous Toaster run self.runbuild() signal.signal(signal.SIGUSR1, lambda sig, frame: None) while True: sigset = signal.sigtimedwait([signal.SIGUSR1], 5) if sigset: for sig in sigset: # Consume each captured pending event self.runbuild() else: # Check for build exceptions self.check_dead_builds()