TheRock/build_tools/tests/fileset_tool_test.py

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

import hashlib
import os
from pathlib import Path
import platform
import shlex
import subprocess
import sys
import tempfile
import unittest

sys.path.insert(0, os.fspath(Path(__file__).parent.parent))
from _therock_utils.hash_util import calculate_hash

FILESET_TOOL = Path(__file__).parent.parent / "fileset_tool.py"

ARTIFACT_DESCRIPTOR_1 = r"""
[options]
unmatched_exclude = "include/foobar.h"

[components.doc]

[components.doc."example/stage"]
include = [
    "**/*.so.1",
]
[components.lib."example/stage"]
"""


def capture(args: list[str | Path], cwd: Path = FILESET_TOOL.parent) -> str:
    args = [str(arg) for arg in args]
    print(f"++ Exec [{cwd}]$ {shlex.join(args)}")
    return subprocess.check_output(
        args, cwd=str(cwd), stdin=subprocess.DEVNULL
    ).decode()


def run_command(args: list[str | Path], cwd: Path = FILESET_TOOL.parent):
    args = [str(arg) for arg in args]
    print(f"++ Exec [{cwd}]$ {shlex.join(args)}")
    return subprocess.check_call(args, cwd=str(cwd), stdin=subprocess.DEVNULL)


def write_text(p: Path, text: str):
    p.parent.mkdir(exist_ok=True, parents=True)
    p.write_text(text)


def is_windows():
    return platform.system() == "Windows"


def fset_executable(f):
    os.fchmod(f.fileno(), os.fstat(f.fileno()).st_mode | 0o111)


def is_executable(path: Path):
    return bool(os.stat(path).st_mode & 0o111)


class FilesetToolTest(unittest.TestCase):
    def setUp(self):
        override_temp = os.getenv("TEST_TMPDIR")
        if override_temp is not None:
            self.temp_context = None
            self.temp_dir = Path(override_temp)
            self.temp_dir.mkdir(parents=True, exist_ok=True)
        else:
            self.temp_context = tempfile.TemporaryDirectory()
            self.temp_dir = Path(self.temp_context.name)

    def tearDown(self):
        if self.temp_context:
            self.temp_context.cleanup()

    # Validates that the happy path flow of creating an artifact, archiving it,
    # expanding and flattening works. This does not exhaustively verify
    # all descriptor options.
    def testSimpleArtifact(self):
        input_dir = self.temp_dir / "input"
        artifact_dir = self.temp_dir / "artifact_dir"
        artifact_archive = self.temp_dir / "artifact.tar.xz"
        descriptor_file = self.temp_dir / "artifact.toml"
        hash_file = self.temp_dir / "artifact.tar.xz.sha256sum"
        flat1_dir = self.temp_dir / "flat1"
        flat2_dir = self.temp_dir / "flat2"
        write_text(descriptor_file, ARTIFACT_DESCRIPTOR_1)

        # One sample file and a symlink, and an executable.
        write_text(
            input_dir / "example" / "stage" / "share" / "doc" / "README.txt",
            "Hello World!",
        )
        Path(input_dir / "example" / "stage" / "share" / "doc" / "README").symlink_to(
            "README.txt"
        )
        if not is_windows():
            with open(
                input_dir / "example" / "stage" / "share" / "doc" / "executable", "wb"
            ) as f:
                f.write(b"Contents")
                fset_executable(f)
        write_text(
            Path(input_dir / "example" / "stage" / "lib" / "libfoobar.so.1"), "foobar"
        )
        write_text(
            Path(input_dir / "example" / "stage" / "include" / "foobar.h"), "foobar"
        )

        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact",
                "--descriptor",
                descriptor_file,
                "--artifact-name",
                "test",
                "--root-dir",
                input_dir,
                "doc",
                artifact_dir,
            ]
        )

        # Validate artifact dir.
        manifest_lines = (
            (artifact_dir / "artifact_manifest.txt").read_text().strip().splitlines()
        )
        self.assertEqual(manifest_lines, ["example/stage"])
        self.assertEqual(
            (
                artifact_dir / "example" / "stage" / "share" / "doc" / "README.txt"
            ).read_text(),
            "Hello World!",
        )
        self.assertEqual(
            os.readlink(
                artifact_dir / "example" / "stage" / "share" / "doc" / "README"
            ),
            "README.txt",
        )
        if not is_windows():
            self.assertTrue(
                is_executable(
                    artifact_dir / "example" / "stage" / "share" / "doc" / "executable"
                )
            )

        # Archive it.
        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-archive",
                artifact_dir,
                "-o",
                artifact_archive,
                "--hash-file",
                hash_file,
            ]
        )

        # Verify digest.
        expected_digest = calculate_hash(artifact_archive, "sha256").hexdigest()
        actual_digest = hash_file.read_text().strip()
        self.assertEqual(expected_digest, actual_digest)

        # Flatten the raw directory and verify.
        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-flatten",
                artifact_dir,
                "-o",
                flat1_dir,
            ]
        )
        self.assertEqual(
            (flat1_dir / "share" / "doc" / "README.txt").read_text(),
            "Hello World!",
        )
        self.assertEqual(
            os.readlink(flat1_dir / "share" / "doc" / "README"),
            "README.txt",
        )
        if not is_windows():
            self.assertTrue(is_executable(flat1_dir / "share" / "doc" / "executable"))

        # Flatten the archive file and verify.
        flatten_output = capture(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-flatten",
                artifact_archive,
                "--verbose",
                "-o",
                flat2_dir,
            ]
        )
        self.assertListEqual(flatten_output.splitlines(), ["example/stage"])
        self.assertEqual(
            (flat2_dir / "share" / "doc" / "README.txt").read_text(),
            "Hello World!",
        )
        self.assertEqual(
            os.readlink(flat2_dir / "share" / "doc" / "README"),
            "README.txt",
        )
        if not is_windows():
            self.assertTrue(is_executable(flat2_dir / "share" / "doc" / "executable"))

    @unittest.skipIf(is_windows(), "Hardlinks not supported the same way on Windows")
    def testHardlinkPreservation(self):
        """Test that hardlinks are preserved through archive/flatten cycle."""
        input_dir = self.temp_dir / "input"
        artifact_dir = self.temp_dir / "artifact_hardlink"
        artifact_archive = self.temp_dir / "artifact_hardlink.tar.xz"
        descriptor_file = self.temp_dir / "hardlink_descriptor.toml"
        flat_dir = self.temp_dir / "flat_hardlink"

        # Create descriptor
        write_text(
            descriptor_file,
            """
[components.lib."example/stage"]
""",
        )

        # Create original file (use lib/ since lib component defaults include lib/**)
        orig_file = input_dir / "example" / "stage" / "lib" / "original.so"
        orig_file.parent.mkdir(parents=True, exist_ok=True)
        orig_file.write_text("hardlink test content")

        # Create hardlink
        link_file = input_dir / "example" / "stage" / "lib" / "hardlink.so"
        os.link(orig_file, link_file)

        # Verify they share inode before archiving
        self.assertEqual(os.stat(orig_file).st_ino, os.stat(link_file).st_ino)

        # Create artifact
        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact",
                "--descriptor",
                descriptor_file,
                "--artifact-name",
                "test",
                "--root-dir",
                input_dir,
                "lib",
                artifact_dir,
            ]
        )

        # Archive it
        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-archive",
                artifact_dir,
                "-o",
                artifact_archive,
            ]
        )

        # Flatten from archive
        run_command(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-flatten",
                artifact_archive,
                "-o",
                flat_dir,
            ]
        )

        # Verify hardlinks are preserved (same inode)
        flat_orig = flat_dir / "lib" / "original.so"
        flat_link = flat_dir / "lib" / "hardlink.so"
        self.assertTrue(flat_orig.exists())
        self.assertTrue(flat_link.exists())
        self.assertEqual(os.stat(flat_orig).st_ino, os.stat(flat_link).st_ino)
        self.assertEqual(flat_orig.read_text(), "hardlink test content")

    def testArtifactFlattenSplit(self):
        """Test artifact-flatten-split discovers and flattens split artifact dirs."""
        artifacts_dir = self.temp_dir / "artifacts"
        output_dir = self.temp_dir / "flat_split"

        # Simulate split artifact directories as produced by the artifact splitter.
        # Each has an artifact_manifest.txt and files under manifest-listed prefixes.
        for suffix in ["generic", "gfx1201"]:
            art_dir = artifacts_dir / f"miopen_lib_{suffix}"
            manifest_prefix = f"miopen_lib_{suffix}/stage"
            write_text(art_dir / "artifact_manifest.txt", manifest_prefix + "\n")
            write_text(
                art_dir / manifest_prefix / "lib" / f"libMIOpen_{suffix}.so",
                f"lib_{suffix}",
            )

        # A second prefix with one variant.
        dev_dir = artifacts_dir / "miopen_dev_generic"
        write_text(dev_dir / "artifact_manifest.txt", "miopen_dev_generic/stage\n")
        write_text(
            dev_dir / "miopen_dev_generic" / "stage" / "include" / "miopen.h",
            "header",
        )

        # A directory that should NOT be discovered (no manifest).
        stray = artifacts_dir / "miopen_lib_stray"
        stray.mkdir(parents=True)
        write_text(stray / "something.txt", "no manifest here")

        # Run artifact-flatten-split with verbose to capture discovered dirs.
        output = capture(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-flatten-split",
                "miopen_lib",
                "miopen_dev",
                "--artifacts-dir",
                artifacts_dir,
                "--verbose",
                "-o",
                output_dir,
            ]
        )

        # Verify verbose output lists discovered dirs.
        self.assertIn("miopen_lib_generic", output)
        self.assertIn("miopen_lib_gfx1201", output)
        self.assertIn("miopen_dev_generic", output)
        self.assertNotIn("miopen_lib_stray", output)

        # Verify flattened files from all three artifact dirs.
        self.assertEqual(
            (output_dir / "lib" / "libMIOpen_generic.so").read_text(),
            "lib_generic",
        )
        self.assertEqual(
            (output_dir / "lib" / "libMIOpen_gfx1201.so").read_text(),
            "lib_gfx1201",
        )
        self.assertEqual(
            (output_dir / "include" / "miopen.h").read_text(),
            "header",
        )

    def testArtifactFlattenSplitNoMatch(self):
        """Test artifact-flatten-split warns when no dirs match."""
        artifacts_dir = self.temp_dir / "empty_artifacts"
        artifacts_dir.mkdir(parents=True)
        output_dir = self.temp_dir / "flat_empty"

        # Should not fail, just warn.
        output = capture(
            [
                sys.executable,
                FILESET_TOOL,
                "artifact-flatten-split",
                "nonexistent_prefix",
                "--artifacts-dir",
                artifacts_dir,
                "-o",
                output_dir,
            ]
        )
        self.assertIn("Warning", output)
        self.assertIn("nonexistent_prefix", output)


if __name__ == "__main__":
    unittest.main()