Fluentbit log test fails sometimes
Summary
The helm test for the testkit seems to fail sometimes with missing logs:
cert-generator-grid-test-cert 12:55:05 ============================= test session starts ==============================
cert-generator-grid-test-cert 12:55:05 platform linux -- Python 3.12.11, pytest-8.4.0, pluggy-1.6.0 -- /usr/local/bin/python3
cert-generator-grid-test-cert 12:55:05 cachedir: //.pytest-cache
cert-generator-grid-test-cert 12:55:05 rootdir: /scripts
cert-generator-grid-test-cert 12:55:05 plugins: mock-3.14.1, cov-6.1.1
cert-generator-grid-test-cert 12:55:06 collecting ... collected 1 item
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 scripts/test_logs.py::test_inspect_service_log
cert-generator-grid-test-cert 12:55:06 -------------------------------- live log call ---------------------------------
cert-generator-grid-test-cert 12:55:06 DEBUG urllib3.connectionpool:connectionpool.py:241 Starting new HTTP connection (1): testkit:8000
cert-generator-grid-test-cert 12:55:06 DEBUG urllib3.connectionpool:connectionpool.py:544 http://testkit:8000 "GET /logs.ndjson HTTP/1.1" 200 None
cert-generator-grid-test-cert 12:55:06 INFO root:test_logs.py:38 found all requested events within 9.0 seconds: ['1749646483.0: Created pod: cert-generator-grid-generate-certificates-fbgbt', '1749646492.0: Job completed']
cert-generator-grid-test-cert 12:55:06 FAILED [100%]
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 =================================== FAILURES ===================================
cert-generator-grid-test-cert 12:55:06 ___________________________ test_inspect_service_log ___________________________
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 def test_inspect_service_log():
cert-generator-grid-test-cert 12:55:06 found = {"log": False, "events": False}
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 lines = []
cert-generator-grid-test-cert 12:55:06 selected_events = []
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 for line in requests.get("http://testkit:8000/logs.ndjson").content.splitlines():
cert-generator-grid-test-cert 12:55:06 try:
cert-generator-grid-test-cert 12:55:06 line_dict = json.loads(line.decode("utf-8"))
cert-generator-grid-test-cert 12:55:06 except json.JSONDecodeError:
cert-generator-grid-test-cert 12:55:06 logging.error("Could not parse line as JSON: '%s'", line)
cert-generator-grid-test-cert 12:55:06 raise
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 if (
cert-generator-grid-test-cert 12:55:06 line_dict.get("kubernetes", {}).get("container_name")
cert-generator-grid-test-cert 12:55:06 == "generate-certificates"
cert-generator-grid-test-cert 12:55:06 ):
cert-generator-grid-test-cert 12:55:06 lines.append(line_dict["log"])
cert-generator-grid-test-cert 12:55:06 if (
cert-generator-grid-test-cert 12:55:06 "Certificates generated and stored" in line_dict["log"]
cert-generator-grid-test-cert 12:55:06 or "Certificates already generated" in line_dict["log"]
cert-generator-grid-test-cert 12:55:06 ):
cert-generator-grid-test-cert 12:55:06 found["log"] = True
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 if (
cert-generator-grid-test-cert 12:55:06 line_dict.get("involvedObject", {}).get("name", None)
cert-generator-grid-test-cert 12:55:06 == "cert-generator-grid-generate-certificates"
cert-generator-grid-test-cert 12:55:06 ):
cert-generator-grid-test-cert 12:55:06 selected_events.append(line_dict)
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 if line_dict.get("message") == "Job completed":
cert-generator-grid-test-cert 12:55:06 logging.info(
cert-generator-grid-test-cert 12:55:06 "found all requested events within %s seconds: %s",
cert-generator-grid-test-cert 12:55:06 selected_events[-1]["date"] - selected_events[0]["date"],
cert-generator-grid-test-cert 12:55:06 [f"{c['date']}: {c['message']}" for c in selected_events],
cert-generator-grid-test-cert 12:55:06 )
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 found["events"] = True
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 if found["log"] and found["events"]:
cert-generator-grid-test-cert 12:55:06 break
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 > assert found["log"], "Did not find expected entries in logs, lines: " + (
cert-generator-grid-test-cert 12:55:06 "\n".join(lines)
cert-generator-grid-test-cert 12:55:06 )
cert-generator-grid-test-cert 12:55:06 E AssertionError: Did not find expected entries in logs, lines:
cert-generator-grid-test-cert 12:55:06 E assert False
cert-generator-grid-test-cert 12:55:06
cert-generator-grid-test-cert 12:55:06 scripts/test_logs.py:49: AssertionErrorSee job: https://gitlab.cta-observatory.org/cta-computing/dpps/aiv/dpps-aiv-toolkit/-/jobs/367187