diff --git a/apps/worker/services/test_analytics/ta_process_flakes.py b/apps/worker/services/test_analytics/ta_process_flakes.py index 61cf26df09..a756dfde0d 100644 --- a/apps/worker/services/test_analytics/ta_process_flakes.py +++ b/apps/worker/services/test_analytics/ta_process_flakes.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from datetime import timedelta import sentry_sdk @@ -33,14 +34,20 @@ def fetch_current_flakes(repo_id: int) -> dict[bytes, Flake]: } -def get_testruns(upload: ReportSession) -> QuerySet[Testrun]: - upload_filter = Q(upload_id=upload.id) - - # we won't process flakes for testruns older than 1 day - return Testrun.objects.filter( - Q(timestamp__gte=timezone.now() - timedelta(days=1)) & upload_filter +def get_all_testruns( + upload_ids: list[int], +) -> dict[int, list[Testrun]]: + """Fetch all testruns for the given upload IDs in a single query and group them by upload_id.""" + testruns = Testrun.objects.filter( + Q(timestamp__gte=timezone.now() - timedelta(days=1)) + & Q(upload_id__in=upload_ids) ).order_by("timestamp") + grouped: dict[int, list[Testrun]] = defaultdict(list) + for testrun in testruns: + grouped[testrun.upload_id].append(testrun) + return grouped + def handle_pass(curr_flakes: dict[bytes, Flake], test_id: bytes): # possible that we expire it and stop caring about it @@ -81,10 +88,12 @@ def handle_failure( @sentry_sdk.trace def process_single_upload( - upload: ReportSession, curr_flakes: dict[bytes, Flake], repo_id: int -): - testruns = get_testruns(upload) - + upload: ReportSession, + curr_flakes: dict[bytes, Flake], + repo_id: int, + testruns: list[Testrun], +) -> list[Testrun]: + """Process a single upload's testruns and return the list of testruns that were modified.""" for testrun in testruns: test_id = bytes(testrun.test_id) match testrun.outcome: @@ -98,7 +107,7 @@ def process_single_upload( case _: continue - Testrun.objects.bulk_update(testruns, ["outcome"]) + return testruns @sentry_sdk.trace @@ -106,7 +115,7 @@ def process_flakes_for_commit(repo_id: int, commit_id: str): log.info( "process_flakes_for_commit: starting processing", ) - uploads = get_relevant_uploads(repo_id, commit_id) + uploads = list(get_relevant_uploads(repo_id, commit_id)) log.info( "process_flakes_for_commit: fetched uploads", @@ -120,13 +129,21 @@ def process_flakes_for_commit(repo_id: int, commit_id: str): extra={"flakes": [flake.test_id.hex() for flake in curr_flakes.values()]}, ) + upload_ids = [upload.id for upload in uploads] + testruns_by_upload = get_all_testruns(upload_ids) + + all_testruns: list[Testrun] = [] for upload in uploads: - process_single_upload(upload, curr_flakes, repo_id) + upload_testruns = testruns_by_upload.get(upload.id, []) + process_single_upload(upload, curr_flakes, repo_id, upload_testruns) + all_testruns.extend(upload_testruns) log.info( "process_flakes_for_commit: processed upload", extra={"upload": upload.id}, ) + Testrun.objects.bulk_update(all_testruns, ["outcome"]) + log.info( "process_flakes_for_commit: bulk creating flakes", extra={"flakes": [flake.test_id.hex() for flake in curr_flakes.values()]},