Files
TheRock/build_tools/tests/artifact_backend_test.py
derek.holloway 7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
Flatten subprojects into a single base monorepo
2026-09-02 16:34:47 -07:00

721 lines
28 KiB
Python

#!/usr/bin/env python
# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT
"""Unit tests for artifact_backend.py."""
import os
import socket
import sys
import tempfile
import unittest
from botocore import UNSIGNED
from pathlib import Path
from unittest import mock
sys.path.insert(0, os.fspath(Path(__file__).parent.parent))
from _therock_utils.artifact_backend import (
ArtifactBackend,
LocalDirectoryBackend,
S3Backend,
create_backend_from_env,
)
from _therock_utils.workflow_outputs import WorkflowOutputRoot
def _make_local_root(run_id="test-run-123", platform="linux"):
return WorkflowOutputRoot.for_local(run_id=run_id, platform=platform)
def _make_s3_root(
bucket="test-bucket",
run_id="test-run-456",
platform="linux",
external_repo="external/",
):
return WorkflowOutputRoot(
bucket=bucket,
external_repo=external_repo,
run_id=run_id,
platform=platform,
)
def _is_public_s3_available():
try:
with socket.create_connection(
("therock-ci-artifacts.s3.amazonaws.com", 443), timeout=5
):
return True
except OSError:
return False
class TestLocalDirectoryBackend(unittest.TestCase):
"""Tests for LocalDirectoryBackend."""
def setUp(self):
"""Create a temporary directory for testing."""
self.temp_dir = tempfile.mkdtemp()
self.output_root = _make_local_root()
self.backend = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=self.output_root,
)
def tearDown(self):
"""Clean up temporary directory."""
import shutil
shutil.rmtree(self.temp_dir, ignore_errors=True)
def test_base_uri(self):
"""Test that base_uri returns the correct path."""
expected = str(Path(self.temp_dir) / "test-run-123-linux")
self.assertEqual(self.backend.base_uri, expected)
def test_base_path_created(self):
"""Test that the base path directory is created."""
self.assertTrue(self.backend.base_path.exists())
self.assertTrue(self.backend.base_path.is_dir())
def test_list_artifacts_empty(self):
"""Test listing artifacts when none exist."""
artifacts = self.backend.list_artifacts()
self.assertEqual(artifacts, [])
def test_list_artifacts_with_files(self):
"""Test listing artifacts with tar.zst and tar.xz files present."""
# Create some test artifact files (both zstd and xz)
(self.backend.base_path / "blas_lib_gfx94X.tar.zst").touch()
(self.backend.base_path / "blas_dev_gfx94X.tar.xz").touch()
(self.backend.base_path / "fft_lib_generic.tar.zst").touch()
# Create a sha256sum file (should be excluded)
(self.backend.base_path / "blas_lib_gfx94X.tar.zst.sha256sum").touch()
# Create a non-artifact file (should be excluded)
(self.backend.base_path / "README.txt").touch()
artifacts = self.backend.list_artifacts()
self.assertEqual(len(artifacts), 3)
self.assertIn("blas_lib_gfx94X.tar.zst", artifacts)
self.assertIn("blas_dev_gfx94X.tar.xz", artifacts)
self.assertIn("fft_lib_generic.tar.zst", artifacts)
def test_list_artifacts_with_name_filter(self):
"""Test filtering artifacts by name prefix."""
(self.backend.base_path / "blas_lib_gfx94X.tar.xz").touch()
(self.backend.base_path / "blas_dev_gfx94X.tar.xz").touch()
(self.backend.base_path / "fft_lib_generic.tar.xz").touch()
# Filter by "blas"
artifacts = self.backend.list_artifacts(name_filter="blas")
self.assertEqual(len(artifacts), 2)
self.assertIn("blas_lib_gfx94X.tar.xz", artifacts)
self.assertIn("blas_dev_gfx94X.tar.xz", artifacts)
# Filter by "fft"
artifacts = self.backend.list_artifacts(name_filter="fft")
self.assertEqual(len(artifacts), 1)
self.assertIn("fft_lib_generic.tar.xz", artifacts)
# Filter with no matches
artifacts = self.backend.list_artifacts(name_filter="nonexistent")
self.assertEqual(len(artifacts), 0)
def test_upload_and_download_artifact_xz(self):
"""Test uploading and downloading a .tar.xz artifact."""
# Create a source file to upload
source_dir = Path(self.temp_dir) / "source"
source_dir.mkdir()
source_file = source_dir / "test_artifact.tar.xz"
source_file.write_text("test artifact content xz")
# Also create a sha256sum file
sha_file = source_dir / "test_artifact.tar.xz.sha256sum"
sha_file.write_text("abc123 test_artifact.tar.xz\n")
# Upload
self.backend.upload_artifact(source_file, "test_artifact.tar.xz")
# Verify it exists in the backend
self.assertTrue(self.backend.artifact_exists("test_artifact.tar.xz"))
self.assertTrue(
(self.backend.base_path / "test_artifact.tar.xz.sha256sum").exists()
)
# Download to a new location
dest_dir = Path(self.temp_dir) / "dest"
dest_dir.mkdir()
dest_file = dest_dir / "downloaded.tar.xz"
self.backend.download_artifact("test_artifact.tar.xz", dest_file)
# Verify content
self.assertTrue(dest_file.exists())
self.assertEqual(dest_file.read_text(), "test artifact content xz")
# Verify sha256sum was also copied
self.assertTrue((dest_dir / "test_artifact.tar.xz.sha256sum").exists())
def test_upload_and_download_artifact_zst(self):
"""Test uploading and downloading a .tar.zst artifact."""
# Create a source file to upload
source_dir = Path(self.temp_dir) / "source_zst"
source_dir.mkdir()
source_file = source_dir / "test_artifact.tar.zst"
source_file.write_text("test artifact content zst")
# Also create a sha256sum file
sha_file = source_dir / "test_artifact.tar.zst.sha256sum"
sha_file.write_text("def456 test_artifact.tar.zst\n")
# Upload
self.backend.upload_artifact(source_file, "test_artifact.tar.zst")
# Verify it exists in the backend
self.assertTrue(self.backend.artifact_exists("test_artifact.tar.zst"))
self.assertTrue(
(self.backend.base_path / "test_artifact.tar.zst.sha256sum").exists()
)
# Download to a new location
dest_dir = Path(self.temp_dir) / "dest_zst"
dest_dir.mkdir()
dest_file = dest_dir / "downloaded.tar.zst"
self.backend.download_artifact("test_artifact.tar.zst", dest_file)
# Verify content
self.assertTrue(dest_file.exists())
self.assertEqual(dest_file.read_text(), "test artifact content zst")
# Verify sha256sum was also copied
self.assertTrue((dest_dir / "test_artifact.tar.zst.sha256sum").exists())
def test_download_nonexistent_artifact(self):
"""Test that downloading a nonexistent artifact raises FileNotFoundError."""
dest_file = Path(self.temp_dir) / "dest" / "nonexistent.tar.xz"
with self.assertRaises(FileNotFoundError):
self.backend.download_artifact("nonexistent.tar.xz", dest_file)
def test_upload_nonexistent_source(self):
"""Test that uploading a nonexistent source raises FileNotFoundError."""
nonexistent = Path(self.temp_dir) / "nonexistent.tar.xz"
with self.assertRaises(FileNotFoundError):
self.backend.upload_artifact(nonexistent, "test.tar.xz")
def test_artifact_exists_xz(self):
"""Test artifact_exists method with .tar.xz."""
self.assertFalse(self.backend.artifact_exists("nonexistent.tar.xz"))
(self.backend.base_path / "exists.tar.xz").touch()
self.assertTrue(self.backend.artifact_exists("exists.tar.xz"))
def test_artifact_exists_zst(self):
"""Test artifact_exists method with .tar.zst."""
self.assertFalse(self.backend.artifact_exists("nonexistent.tar.zst"))
(self.backend.base_path / "exists.tar.zst").touch()
self.assertTrue(self.backend.artifact_exists("exists.tar.zst"))
def test_copy_artifact_between_local_backends(self):
"""Test copy_artifact copies files between two local backends."""
source = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="source-run"),
)
dest = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="dest-run"),
)
# Create artifact and sha256sum in source
artifact_key = "test_lib_generic.tar.zst"
(source.base_path / artifact_key).write_bytes(b"test content")
(source.base_path / f"{artifact_key}.sha256sum").write_text(
"abc123 test_lib_generic.tar.zst\n"
)
# Copy to dest
dest.copy_artifact(artifact_key, source)
# Verify artifact and sha256sum were both copied
self.assertTrue(dest.artifact_exists(artifact_key))
self.assertEqual((dest.base_path / artifact_key).read_bytes(), b"test content")
self.assertTrue((dest.base_path / f"{artifact_key}.sha256sum").exists())
def test_copy_artifact_without_sha256sum(self):
"""Test copy_artifact works when no sha256sum file exists."""
source = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="source-run"),
)
dest = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="dest-run"),
)
# Create artifact without sha256sum
artifact_key = "test_lib_generic.tar.zst"
(source.base_path / artifact_key).write_bytes(b"test content")
# Copy should succeed without error
dest.copy_artifact(artifact_key, source)
self.assertTrue(dest.artifact_exists(artifact_key))
self.assertFalse((dest.base_path / f"{artifact_key}.sha256sum").exists())
def test_copy_artifact_nonexistent_raises(self):
"""Test copy_artifact raises FileNotFoundError for missing source artifact."""
source = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="source-run"),
)
dest = LocalDirectoryBackend(
staging_dir=Path(self.temp_dir),
output_root=_make_local_root(run_id="dest-run"),
)
with self.assertRaises(FileNotFoundError):
dest.copy_artifact("nonexistent.tar.zst", source)
def test_copy_artifact_wrong_backend_type_raises(self):
"""Test copy_artifact raises TypeError when source is a different backend type."""
s3_source = S3Backend(
output_root=_make_s3_root(run_id="source-run"),
)
with self.assertRaises(TypeError):
self.backend.copy_artifact("test.tar.zst", s3_source)
class TestS3Backend(unittest.TestCase):
"""Tests for S3Backend with mocked boto3 client."""
def setUp(self):
"""Set up S3Backend with mocked client."""
self.output_root = _make_s3_root()
self.backend = S3Backend(output_root=self.output_root)
def test_base_uri(self):
"""Test that base_uri returns the correct S3 URI."""
expected = "s3://test-bucket/external/test-run-456-linux"
self.assertEqual(self.backend.base_uri, expected)
def test_s3_prefix(self):
"""Test that s3_prefix is constructed correctly."""
self.assertEqual(self.backend.s3_prefix, "external/test-run-456-linux")
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_list_artifacts(self, mock_client_prop):
"""Test listing S3 artifacts (both zstd and xz)."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
# Mock paginator with both zstd and xz artifacts
mock_paginator = mock.MagicMock()
mock_client.get_paginator.return_value = mock_paginator
mock_paginator.paginate.return_value = [
{
"Contents": [
{"Key": "external/test-run-456-linux/blas_lib_gfx94X.tar.zst"},
{"Key": "external/test-run-456-linux/blas_dev_gfx94X.tar.xz"},
{
"Key": "external/test-run-456-linux/blas_lib_gfx94X.tar.zst.sha256sum"
},
]
}
]
artifacts = self.backend.list_artifacts()
self.assertEqual(len(artifacts), 2)
self.assertIn("blas_lib_gfx94X.tar.zst", artifacts)
self.assertIn("blas_dev_gfx94X.tar.xz", artifacts)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_list_artifacts_with_filter(self, mock_client_prop):
"""Test listing S3 artifacts with name filter."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
mock_paginator = mock.MagicMock()
mock_client.get_paginator.return_value = mock_paginator
mock_paginator.paginate.return_value = [
{
"Contents": [
{"Key": "external/test-run-456-linux/blas_lib_gfx94X.tar.xz"},
{"Key": "external/test-run-456-linux/fft_lib_generic.tar.xz"},
]
}
]
artifacts = self.backend.list_artifacts(name_filter="blas")
self.assertEqual(len(artifacts), 1)
self.assertIn("blas_lib_gfx94X.tar.xz", artifacts)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_download_artifact_xz(self, mock_client_prop):
"""Test downloading a .tar.xz artifact from S3."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
with tempfile.TemporaryDirectory() as temp_dir:
dest_path = Path(temp_dir) / "downloaded.tar.xz"
self.backend.download_artifact("test.tar.xz", dest_path)
mock_client.download_file.assert_called_once_with(
"test-bucket",
"external/test-run-456-linux/test.tar.xz",
str(dest_path),
)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_download_artifact_zst(self, mock_client_prop):
"""Test downloading a .tar.zst artifact from S3."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
with tempfile.TemporaryDirectory() as temp_dir:
dest_path = Path(temp_dir) / "downloaded.tar.zst"
self.backend.download_artifact("test.tar.zst", dest_path)
mock_client.download_file.assert_called_once_with(
"test-bucket",
"external/test-run-456-linux/test.tar.zst",
str(dest_path),
)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_upload_artifact_xz(self, mock_client_prop):
"""Test uploading a .tar.xz artifact to S3."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
with tempfile.TemporaryDirectory() as temp_dir:
source_path = Path(temp_dir) / "test.tar.xz"
source_path.touch()
self.backend.upload_artifact(source_path, "test.tar.xz")
mock_client.upload_file.assert_called_once_with(
str(source_path),
"test-bucket",
"external/test-run-456-linux/test.tar.xz",
)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_upload_artifact_zst(self, mock_client_prop):
"""Test uploading a .tar.zst artifact to S3."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
with tempfile.TemporaryDirectory() as temp_dir:
source_path = Path(temp_dir) / "test.tar.zst"
source_path.touch()
self.backend.upload_artifact(source_path, "test.tar.zst")
mock_client.upload_file.assert_called_once_with(
str(source_path),
"test-bucket",
"external/test-run-456-linux/test.tar.zst",
)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_artifact_exists_true(self, mock_client_prop):
"""Test artifact_exists when artifact exists."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
self.assertTrue(self.backend.artifact_exists("exists.tar.xz"))
mock_client.head_object.assert_called_once()
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_artifact_exists_false(self, mock_client_prop):
"""Test artifact_exists when artifact does not exist."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
mock_client.head_object.side_effect = Exception("Not found")
self.assertFalse(self.backend.artifact_exists("nonexistent.tar.xz"))
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_copy_artifact_same_bucket(self, mock_client_prop):
"""Test S3 server-side copy within the same bucket."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
# sha256sum exists in source
mock_client.head_object.return_value = {}
source = S3Backend(
output_root=_make_s3_root(
bucket="test-bucket", run_id="source-run", external_repo=""
)
)
dest = S3Backend(
output_root=_make_s3_root(
bucket="test-bucket", run_id="dest-run", external_repo=""
)
)
# Share the mock client
source._s3_client = mock_client
dest.copy_artifact("artifact_lib_generic.tar.zst", source)
# Verify both artifact and sha256sum were copied
self.assertEqual(mock_client.copy.call_count, 2)
mock_client.copy.assert_any_call(
{
"Bucket": "test-bucket",
"Key": "source-run-linux/artifact_lib_generic.tar.zst",
},
"test-bucket",
"dest-run-linux/artifact_lib_generic.tar.zst",
)
mock_client.copy.assert_any_call(
{
"Bucket": "test-bucket",
"Key": "source-run-linux/artifact_lib_generic.tar.zst.sha256sum",
},
"test-bucket",
"dest-run-linux/artifact_lib_generic.tar.zst.sha256sum",
)
@mock.patch.object(S3Backend, "s3_client", new_callable=mock.PropertyMock)
def test_copy_artifact_cross_bucket(self, mock_client_prop):
"""Test S3 server-side copy across different buckets."""
mock_client = mock.MagicMock()
mock_client_prop.return_value = mock_client
# sha256sum does not exist in source
mock_client.head_object.side_effect = Exception("Not found")
source = S3Backend(
output_root=_make_s3_root(
bucket="therock-ci-artifacts",
run_id="source-run",
external_repo="",
)
)
dest = S3Backend(
output_root=_make_s3_root(
bucket="therock-ci-artifacts-external",
run_id="dest-run",
external_repo="ROCm-rocm-libraries/",
)
)
source._s3_client = mock_client
dest.copy_artifact("artifact_lib_generic.tar.zst", source)
# Only artifact copied (no sha256sum in source)
mock_client.copy.assert_called_once_with(
{
"Bucket": "therock-ci-artifacts",
"Key": "source-run-linux/artifact_lib_generic.tar.zst",
},
"therock-ci-artifacts-external",
"ROCm-rocm-libraries/dest-run-linux/artifact_lib_generic.tar.zst",
)
def test_copy_artifact_wrong_backend_type_raises(self):
"""Test copy_artifact raises TypeError when source is a different backend type."""
import tempfile
local_source = LocalDirectoryBackend(
staging_dir=Path(tempfile.mkdtemp()),
output_root=_make_local_root(run_id="source-run"),
)
with self.assertRaises(TypeError):
self.backend.copy_artifact("test.tar.zst", local_source)
class TestS3BackendCredentials(unittest.TestCase):
"""Live tests for S3Backend credential resolution.
These exercise the real boto3 credential chain (no mocks) to verify:
- UNSIGNED fallback works for public bucket reads without credentials
- AWS_SHARED_CREDENTIALS_FILE is used (so fork PRs can upload artifacts)
"""
# Env vars that could provide credentials to boto3's default chain.
_CREDENTIAL_ENV_VARS = [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"AWS_SHARED_CREDENTIALS_FILE",
"AWS_CONFIG_FILE",
"AWS_PROFILE",
]
def _make_backend(self):
output_root = WorkflowOutputRoot(
bucket="therock-ci-artifacts",
external_repo="",
run_id="placeholder",
platform="linux",
)
return S3Backend(output_root=output_root)
@unittest.skipUnless(
_is_public_s3_available(),
"Public S3 endpoint is not reachable (sandbox with no network access?)",
)
def test_list_objects_unsigned(self):
"""Listing objects in a public bucket must succeed without credentials."""
# Point config/credentials files at nonexistent paths so that
# ~/.aws/credentials on developer machines is not picked up.
env_overrides = {
"AWS_CONFIG_FILE": "/nonexistent/aws/config",
"AWS_SHARED_CREDENTIALS_FILE": "/nonexistent/aws/credentials",
}
with mock.patch.dict(os.environ, env_overrides, clear=False):
for var in self._CREDENTIAL_ENV_VARS:
if var not in env_overrides:
os.environ.pop(var, None)
backend = self._make_backend()
config = backend.s3_client._client_config
self.assertEqual(
config.signature_version,
UNSIGNED,
"Client should be UNSIGNED when all credentials are cleared",
)
# A real list call against the public bucket should succeed
response = backend.s3_client.list_objects_v2(
Bucket="therock-ci-artifacts",
MaxKeys=1,
)
self.assertIn("Contents", response)
self.assertGreater(len(response["Contents"]), 0)
def test_shared_credentials_file_produces_authenticated_client(self):
"""AWS_SHARED_CREDENTIALS_FILE must produce an authenticated client.
Fork PRs provide credentials via a mounted credentials.ini file,
not via AWS_ACCESS_KEY_ID/SECRET/TOKEN env vars. This test would
fail with the old manual 3-env-var check that ignored
AWS_SHARED_CREDENTIALS_FILE.
"""
# Write a minimal credentials file. Values are intentionally not
# real AWS keys — boto3 loads any syntactically valid values.
with tempfile.NamedTemporaryFile(
mode="w", suffix=".ini", delete=False
) as creds_file:
creds_file.write(
"[default]\n"
"aws_access_key_id = test\n"
"aws_secret_access_key = test\n"
)
creds_path = creds_file.name
try:
env_overrides = {
"AWS_SHARED_CREDENTIALS_FILE": creds_path,
"AWS_CONFIG_FILE": "/nonexistent/aws/config",
}
env_clear = [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"AWS_PROFILE",
]
with mock.patch.dict(os.environ, env_overrides, clear=False):
for var in env_clear:
os.environ.pop(var, None)
backend = self._make_backend()
config = backend.s3_client._client_config
self.assertNotEqual(
config.signature_version,
UNSIGNED,
"Client must NOT use UNSIGNED when credentials are "
"available via AWS_SHARED_CREDENTIALS_FILE. The old "
"manual env-var check ignored this credential source, "
"breaking uploads from fork PRs.",
)
finally:
os.unlink(creds_path)
class TestCreateBackendFromEnv(unittest.TestCase):
"""Tests for create_backend_from_env factory function."""
def test_local_backend_from_env(self):
"""Test that LocalDirectoryBackend is created when env var is set."""
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(
os.environ,
{
"THEROCK_LOCAL_STAGING_DIR": temp_dir,
"THEROCK_RUN_ID": "env-run-id",
"THEROCK_PLATFORM": "windows",
},
):
backend = create_backend_from_env()
self.assertIsInstance(backend, LocalDirectoryBackend)
self.assertIn("env-run-id", backend.base_uri)
self.assertIn("windows", backend.base_uri)
def test_local_backend_with_overrides(self):
"""Test LocalDirectoryBackend with explicit run_id and platform."""
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(
os.environ,
{
"THEROCK_LOCAL_STAGING_DIR": temp_dir,
},
):
backend = create_backend_from_env(
run_id="override-run", platform="override-platform"
)
self.assertIsInstance(backend, LocalDirectoryBackend)
self.assertIn("override-run", backend.base_uri)
self.assertIn("override-platform", backend.base_uri)
@mock.patch("_therock_utils.workflow_outputs._retrieve_bucket_info")
def test_s3_backend_when_no_local_dir(self, mock_retrieve):
"""Test that S3Backend is created when THEROCK_LOCAL_STAGING_DIR is not set."""
mock_retrieve.return_value = ("", "test-bucket")
with mock.patch.dict(
os.environ,
{"THEROCK_RUN_ID": "s3-run-id"},
clear=True,
):
backend = create_backend_from_env()
self.assertIsInstance(backend, S3Backend)
self.assertEqual(backend.bucket, "test-bucket")
self.assertIn("s3-run-id", backend.s3_prefix)
@mock.patch("_therock_utils.workflow_outputs._retrieve_bucket_info")
def test_s3_backend_with_github_repository_override(self, mock_retrieve):
"""Test that explicit github_repository overrides GITHUB_REPOSITORY env var.
When running on a fork (GITHUB_REPOSITORY=SomeUser/TheRock), passing
github_repository="ROCm/TheRock" should use the main repo's bucket.
"""
mock_retrieve.return_value = ("", "therock-ci-artifacts")
with mock.patch.dict(
os.environ,
{
"THEROCK_RUN_ID": "s3-run-id",
"GITHUB_REPOSITORY": "SomeUser/TheRock",
},
clear=True,
):
backend = create_backend_from_env(github_repository="ROCm/TheRock")
self.assertIsInstance(backend, S3Backend)
self.assertEqual(backend.bucket, "therock-ci-artifacts")
# ROCm/TheRock has no external_repo prefix
self.assertNotIn("SomeUser", backend.s3_prefix)
if __name__ == "__main__":
unittest.main()