derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
836 lines
33 KiB
Python
Executable File
836 lines
33 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright (C) 2017 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import dataclasses as dc
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
import bz2
|
|
|
|
from collections import namedtuple
|
|
from platform import system, machine
|
|
|
|
|
|
# The format for the deps below is the following:
|
|
# (target_folder, source_url, sha1, target_os, target_arch)
|
|
# |source_url| can be either a git repo or a http url.
|
|
# If a git repo, |checksum| is the SHA1 committish that will be checked out.
|
|
# If a http url, |checksum| is the SHA256 of the downloaded file.
|
|
# If the url is a .zip, .tgz, or .tbz2 file it will be automatically deflated under
|
|
# |target_folder|, taking care of stripping the root folder if it's a single
|
|
# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
|
|
# instead just buildtools/protobuf).
|
|
# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
|
|
# |target_arch| is either 'x64', 'arm64' or 'all'
|
|
# in both cases the dep only applies on matching platforms
|
|
# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
|
|
# case of MacOS universal binaries.
|
|
@dc.dataclass
|
|
class Dependency:
|
|
target_folder: str
|
|
source_url: str
|
|
checksum: str
|
|
target_os: str
|
|
target_arch: str
|
|
submodules: bool = False
|
|
|
|
|
|
# This is to remove old directories when build tools get {re,}moved. This is to
|
|
# avoid accidentally referring to stale dir in custom user scripts.
|
|
CLEANUP_OLD_DIRS = [
|
|
'buildtools/nodejs', # Moved to buildtools/{mac,linux64}/nodejs
|
|
'buildtools/emsdk', # Moved to buildtools/{mac,linux64}/emsdk
|
|
'buildtools/test_data', # Moved to test/data by r.android.com/1539381 .
|
|
'buildtools/d8', # Removed by r.android.com/1424334 .
|
|
|
|
# Build tools moved to third_party/ by r.android.com/2327602 .
|
|
'buildtools/mac/clang-format',
|
|
'buildtools/mac/gn',
|
|
'buildtools/mac/ninja',
|
|
'buildtools/linux64/clang-format',
|
|
'buildtools/linux64/gn',
|
|
'buildtools/linux64/ninja',
|
|
'buildtools/win/clang-format.exe',
|
|
'buildtools/win/gn.exe',
|
|
'buildtools/win/ninja.exe',
|
|
]
|
|
|
|
# Dependencies required to build code on the host or when targeting desktop OS.
|
|
BUILD_DEPS_TOOLCHAIN_HOST = [
|
|
# GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
|
|
# git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
|
|
Dependency(
|
|
'third_party/gn/gn',
|
|
'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
|
|
'9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
|
|
'darwin', 'x64'),
|
|
Dependency(
|
|
'third_party/gn/gn',
|
|
'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
|
|
'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
|
|
'darwin', 'arm64'),
|
|
Dependency(
|
|
'third_party/gn/gn',
|
|
'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
|
|
'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'third_party/gn/gn',
|
|
'https://storage.googleapis.com/perfetto/gn-linux-arm64-1968-0725d782',
|
|
'c2a372cd4f911028d8bc351fbf24835c9b1194fcc92beadf6c5a2b3addae973c',
|
|
'linux', 'arm64'),
|
|
Dependency(
|
|
'third_party/gn/gn.exe',
|
|
'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
|
|
'001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
|
|
'windows', 'x64'),
|
|
|
|
# clang-format
|
|
# From
|
|
# https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.arm64.sha1
|
|
Dependency(
|
|
'third_party/clang-format/clang-format',
|
|
'https://storage.googleapis.com/chromium-clang-format/5553d7a3d912b7d49381ad68c9a56740601a57a0',
|
|
'e077990b2ea6807f6abc71b4cf1e487719d7e40574baddd2b51187fdcc8db803',
|
|
'darwin', 'arm64'),
|
|
# From
|
|
# https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.x64.sha1
|
|
Dependency(
|
|
'third_party/clang-format/clang-format',
|
|
'https://storage.googleapis.com/chromium-clang-format/87d69aeff220c916b85b5d6d162fa5668aa9d64f',
|
|
'71179a8788a009cfd589636d50f0eb9f95f58b0cfda4351430bed7c0a48c080b',
|
|
'darwin', 'x64'),
|
|
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
|
|
Dependency(
|
|
'third_party/clang-format/clang-format',
|
|
'https://storage.googleapis.com/chromium-clang-format/1facab3101fc6da6b9467543f27f0622b966bc19',
|
|
'5e459118d8ac825452e9e1f2717e4de5a36399dc6cc6aec7ec483ad27a0c927e',
|
|
'linux', 'x64'),
|
|
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
|
|
Dependency(
|
|
'third_party/clang-format/clang-format.exe',
|
|
'https://storage.googleapis.com/chromium-clang-format/2e569921b9884daf157021d6ae9e21991cd6cf81',
|
|
'2227376ada89ea832995b832222b722a27c4d5d8d59e9c4d7842877f99a1e30d',
|
|
'windows', 'x64'),
|
|
|
|
# Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
|
|
Dependency(
|
|
'buildtools/clang_format/script',
|
|
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/clang/tools/clang-format.git',
|
|
'f97059df7f8b205064625cdb5f97b56668a125ef', 'all', 'all'),
|
|
|
|
# Ninja
|
|
Dependency(
|
|
'third_party/ninja/ninja',
|
|
'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
|
|
'36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
|
|
'darwin', 'all'),
|
|
Dependency(
|
|
'third_party/ninja/ninja',
|
|
'https://storage.googleapis.com/perfetto/ninja-linux64-182',
|
|
'54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'third_party/ninja/ninja.exe',
|
|
'https://storage.googleapis.com/perfetto/ninja-win-182',
|
|
'09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
|
|
'windows', 'x64'),
|
|
Dependency(
|
|
'third_party/ninja/ninja',
|
|
'https://storage.googleapis.com/perfetto/ninja-linux-arm64-1111',
|
|
'05031a734ec4310a51b2cfe9f0096b26fce25ab4ff19e5b51abe6371de066cc5',
|
|
'linux', 'arm64'),
|
|
|
|
# Keep the revision in sync with Chrome's PACKAGE_VERSION in
|
|
# tools/clang/scripts/update.py.
|
|
Dependency(
|
|
'buildtools/linux64/clang.tgz',
|
|
'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
|
|
'6741cc1083f935795330b6e04617ac891a7b5d2b5647b664c5b0fccc354adb43',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'buildtools/win/clang.tgz',
|
|
'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-19-init-2941-ga0b3dbaf-22.tgz',
|
|
'f627080ed53d4c156f089323e04fa3690c8bb459110b62cd1952b0e1f0755987',
|
|
'windows', 'x64'),
|
|
]
|
|
|
|
BUILD_DEPS_HOST = [
|
|
# Keep in sync with Android's //external/googletest/METADATA.
|
|
Dependency(
|
|
'buildtools/googletest',
|
|
'https://android.googlesource.com/platform/external/googletest.git',
|
|
'609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
|
|
|
|
# Keep in sync with Chromium's //third_party/protobuf.
|
|
Dependency(
|
|
'buildtools/protobuf',
|
|
# If you revert the below version back to an earlier version of
|
|
# protobuf, make sure to revert the changes to
|
|
# //gn/standalone/protoc.py as well.
|
|
#
|
|
# This comment can be removed with protobuf is next upreved.
|
|
'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
|
|
'f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c', # refs/tags/v21.12
|
|
'all',
|
|
'all'),
|
|
|
|
# libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
|
|
# lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
|
|
Dependency(
|
|
'buildtools/libcxx',
|
|
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
|
|
'852bc6746f45add53fec19f3a29280e69e358d44', 'all', 'all'),
|
|
Dependency(
|
|
'buildtools/libcxxabi',
|
|
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
|
|
'a37a3aa431f132b02a58656f13984d51098330a2', 'all', 'all'),
|
|
Dependency(
|
|
'buildtools/libunwind',
|
|
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
|
|
'419b03c0b8f20d6da9ddcb0d661a94a97cdd7dad', 'all', 'all'),
|
|
|
|
# Keep in sync with chromium DEPS.
|
|
Dependency(
|
|
'buildtools/libfuzzer',
|
|
'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
|
|
'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
|
|
|
|
# Benchmarking tool.
|
|
Dependency(
|
|
'buildtools/benchmark',
|
|
'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
|
|
'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
|
|
|
|
# Libbacktrace, for stacktraces in Linux/Android debug builds.
|
|
# From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
|
|
Dependency(
|
|
'buildtools/libbacktrace.zip',
|
|
'https://storage.googleapis.com/perfetto/libbacktrace-14818b7783eeb9a56c3f0fca78cefd3143f8c5f6.zip',
|
|
'0d09295938155aa84d9a6049f63df8cd2def3a28302b3550ea3ead9100b3d086',
|
|
'all', 'all'),
|
|
|
|
# Sqlite for the trace processing library.
|
|
# This is the amalgamated source whose compiled output is meant to be faster.
|
|
# We still pull the full source for the extensions (which are not available
|
|
# in the amalgamation).
|
|
# If updating the version, also update bazel/deps.bzl.
|
|
Dependency(
|
|
'buildtools/sqlite.zip',
|
|
'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3440200.zip',
|
|
'833be89b53b3be8b40a2e3d5fedb635080e3edb204957244f3d6987c2bb2345f',
|
|
'all', 'all'),
|
|
Dependency(
|
|
'buildtools/sqlite_src',
|
|
'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
|
|
'c8f9803dc32bfee78a9ca2b1abbe39499729219b', # refs/tags/version-3.44.2.
|
|
'all',
|
|
'all'),
|
|
|
|
# JsonCpp for legacy json import. Used only by the trace processor in
|
|
# standalone builds.
|
|
# If updating the version, also update bazel/deps.bzl.
|
|
Dependency(
|
|
'buildtools/jsoncpp',
|
|
'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
|
|
'6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2', # refs/tags/1.9.3.
|
|
'all',
|
|
'all'),
|
|
|
|
# Archive with only the demangling sources from llvm-project.
|
|
# See tools/repackage_llvm_demangler.sh on how to update this.
|
|
# File suffix is the git reference to the commit at which we rearchived the
|
|
# sources, as hosted on https://llvm.googlesource.com/llvm-project.
|
|
# If updating the version, also update bazel/deps.bzl.
|
|
Dependency(
|
|
'buildtools/llvm-project.tgz',
|
|
'https://storage.googleapis.com/perfetto/llvm-project-617a15a9eac96088ae5e9134248d8236e34b91b1.tgz',
|
|
'7e2541446a27f2a09a84520da7bc93cd71749ba0f17318f2d5291fbf45b97956',
|
|
'all', 'all'),
|
|
|
|
# These dependencies are for libunwindstack, which is used by src/profiling.
|
|
Dependency('buildtools/android-core',
|
|
'https://android.googlesource.com/platform/system/core.git',
|
|
'9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
|
|
Dependency(
|
|
'buildtools/android-unwinding',
|
|
'https://android.googlesource.com/platform/system/unwinding.git',
|
|
'4b59ea8471e89d01300481a92de3230b79b6d7c7', 'all', 'all'),
|
|
Dependency('buildtools/android-logging',
|
|
'https://android.googlesource.com/platform/system/logging.git',
|
|
'7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
|
|
Dependency('buildtools/android-libbase',
|
|
'https://android.googlesource.com/platform/system/libbase.git',
|
|
'78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
|
|
Dependency(
|
|
'buildtools/android-libprocinfo',
|
|
'https://android.googlesource.com/platform/system/libprocinfo.git',
|
|
'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
|
|
Dependency('buildtools/lzma',
|
|
'https://android.googlesource.com/platform/external/lzma.git',
|
|
'7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
|
|
Dependency('buildtools/zstd',
|
|
'https://android.googlesource.com/platform/external/zstd.git',
|
|
'77211fcc5e08c781734a386402ada93d0d18d093', 'all', 'all'),
|
|
Dependency('buildtools/bionic',
|
|
'https://android.googlesource.com/platform/bionic.git',
|
|
'a0d0355105cb9d4a4b5384897448676133d7b8e2', 'all', 'all'),
|
|
|
|
# Zlib used both in the tracing binaries, as well as the trace processor and
|
|
# assorted tools.
|
|
# If updating the version, also update bazel/deps.bzl.
|
|
Dependency('buildtools/zlib',
|
|
'https://android.googlesource.com/platform/external/zlib.git',
|
|
'6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
|
|
|
|
# Linenoise, used only by trace_processor in standalone builds.
|
|
# If updating the version, also update bazel/deps.bzl.
|
|
Dependency('buildtools/linenoise',
|
|
'https://fuchsia.googlesource.com/third_party/linenoise.git',
|
|
'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
|
|
|
|
# Bloaty, used to investigate binary size
|
|
Dependency(
|
|
'buildtools/bloaty.zip',
|
|
'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
|
|
'2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
|
|
'darwin', 'all'),
|
|
]
|
|
|
|
# Dependencies required to build Android code.
|
|
# URLs and SHA1s taken from:
|
|
# - https://dl.google.com/android/repository/repository-11.xml
|
|
# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
|
|
BUILD_DEPS_ANDROID = [
|
|
# Android NDK
|
|
Dependency(
|
|
'buildtools/ndk.zip',
|
|
'https://dl.google.com/android/repository/android-ndk-r26c-darwin.zip',
|
|
'312756dfcbdbf389d35d651e17ca98683bd36cb83cc7bf7ad51cac5c06bd064b',
|
|
'darwin', 'all'),
|
|
Dependency(
|
|
'buildtools/ndk.zip',
|
|
'https://dl.google.com/android/repository/android-ndk-r26c-linux.zip',
|
|
'6d6e659834d28bb24ba7ae66148ad05115ebbad7dabed1af9b3265674774fcf6',
|
|
'linux', 'x64'),
|
|
]
|
|
|
|
# Dependencies required to run Android tests.
|
|
TEST_DEPS_ANDROID = [
|
|
# Android emulator images.
|
|
Dependency(
|
|
'buildtools/aosp-arm.zip',
|
|
'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
|
|
'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
|
|
'all', 'all'),
|
|
|
|
# platform-tools.zip contains adb binaries.
|
|
Dependency(
|
|
'buildtools/android_sdk/platform-tools.zip',
|
|
'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
|
|
'98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
|
|
'darwin', 'all'),
|
|
Dependency(
|
|
'buildtools/android_sdk/platform-tools.zip',
|
|
'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
|
|
'90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
|
|
'linux', 'x64'),
|
|
|
|
# Android emulator binaries.
|
|
Dependency(
|
|
'buildtools/emulator',
|
|
'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
|
|
'4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
|
|
]
|
|
|
|
# This variable is updated by tools/roll-catapult-trace-viewer.
|
|
CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
|
|
|
|
TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
|
|
|
|
UI_DEPS = [
|
|
Dependency(
|
|
'buildtools/mac/nodejs.tgz',
|
|
'https://storage.googleapis.com/chromium-nodejs/16.13.0/31859fc1fa0994a95f44f09c367d6ff63607cfde',
|
|
'd9cf1f36b16e08ce52cfbdef427c6596eed2bd2f3608a79112d25b4ec8f75753',
|
|
'darwin', 'arm64'),
|
|
Dependency(
|
|
'buildtools/mac/nodejs.tgz',
|
|
'https://storage.googleapis.com/chromium-nodejs/16.13.0/16dfd094763b71988933a31735f9dea966f9abd6',
|
|
'065ced7e93f0e26276cb74d9688706674323865c4a8fc89e4adfa6868d4ebb4d',
|
|
'darwin', 'x64'),
|
|
Dependency(
|
|
'buildtools/linux64/nodejs.tgz',
|
|
'https://storage.googleapis.com/chromium-nodejs/16.13.0/ab9544e24e752d3d17f335fb7b2055062e582d11',
|
|
'3b5ca150a55f3aadfa18f3a10a4495aaf9653614ba3e460647170fef5287ec4f',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'buildtools/mac/emsdk.tgz',
|
|
'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
|
|
'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
|
|
'darwin', 'all'),
|
|
Dependency(
|
|
'buildtools/linux64/emsdk.tgz',
|
|
'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
|
|
'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'buildtools/catapult_trace_viewer.tgz',
|
|
'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
|
|
% CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
|
|
Dependency(
|
|
'buildtools/typefaces.tgz',
|
|
'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
|
|
TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all'),
|
|
|
|
Dependency(
|
|
'third_party/pnpm/pnpm',
|
|
'https://storage.googleapis.com/perfetto/pnpm-linux-arm64-8.6.3',
|
|
'ac76e9ab6a770479f93c1a2bf978d72636dbcb02608554378cf30075a78a22ac',
|
|
'linux', 'arm64'),
|
|
Dependency(
|
|
'third_party/pnpm/pnpm',
|
|
'https://storage.googleapis.com/perfetto/pnpm-linux-x64-8.6.3',
|
|
'5a58ccd78d44faac138d901976a7a8917c0f2a2f83743cfdd895fcd0bb6aa135',
|
|
'linux', 'x64'),
|
|
Dependency(
|
|
'third_party/pnpm/pnpm',
|
|
'https://storage.googleapis.com/perfetto/pnpm-macos-arm64-8.6.3',
|
|
'f527713d3183e30cfbfd7fd6403ceed730831c53649e50c979961eab3b2cf866',
|
|
'darwin', 'arm64'),
|
|
Dependency(
|
|
'third_party/pnpm/pnpm',
|
|
'https://storage.googleapis.com/perfetto/pnpm-macos-x64-8.6.3',
|
|
'6b425f7f0342341e9ee9427a9a2be2c89936c4a04efe6125f7af667eb02b10c1',
|
|
'darwin', 'x64'),
|
|
]
|
|
|
|
# Dependencies to build gRPC.
|
|
GRPC_DEPS = [
|
|
Dependency(
|
|
'buildtools/grpc/src',
|
|
'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
|
|
'4795c5e69b25e8c767b498bea784da0ef8c96fd5', 'all', 'all', True),
|
|
]
|
|
|
|
# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
|
|
# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
|
|
BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
|
|
Dependency(
|
|
'buildtools/debian_sid_arm-sysroot.tgz',
|
|
'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
|
|
'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
|
|
'linux', 'all'),
|
|
Dependency(
|
|
'buildtools/debian_sid_arm64-sysroot.tgz',
|
|
'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
|
|
'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
|
|
'linux', 'all'),
|
|
]
|
|
|
|
ALL_DEPS = (
|
|
BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
|
|
TEST_DEPS_ANDROID + UI_DEPS)
|
|
|
|
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
UI_DIR = os.path.join(ROOT_DIR, 'ui')
|
|
TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
|
|
NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
|
|
TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
|
|
|
|
|
|
def CheckCallRetry(*args, **kwargs):
|
|
""" Like subprocess.check_call, with retries up to 5 times. """
|
|
MAX_ATTEMPTS = 5
|
|
for attempt in range(1, MAX_ATTEMPTS + 1):
|
|
try:
|
|
return subprocess.check_call(*args, **kwargs)
|
|
except subprocess.CalledProcessError as error:
|
|
if attempt == MAX_ATTEMPTS:
|
|
raise error
|
|
else:
|
|
logging.error(error)
|
|
time.sleep(attempt * 3)
|
|
|
|
|
|
def DownloadURL(url, out_file):
|
|
CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
|
|
|
|
|
|
def GetArch():
|
|
arch = machine()
|
|
if arch == 'arm64':
|
|
return 'arm64'
|
|
elif arch == 'aarch64':
|
|
return 'arm64'
|
|
else:
|
|
# Assume everything else is x64 matching previous behaviour.
|
|
return 'x64'
|
|
|
|
|
|
def ReadFile(path):
|
|
if not os.path.exists(path):
|
|
return None
|
|
with open(path) as f:
|
|
return f.read().strip()
|
|
|
|
|
|
def MkdirRecursive(path):
|
|
# Works with both relative and absolute paths
|
|
cwd = '/' if path.startswith('/') else ROOT_DIR
|
|
for part in path.split('/'):
|
|
cwd = os.path.join(cwd, part)
|
|
if not os.path.exists(cwd):
|
|
os.makedirs(cwd)
|
|
else:
|
|
assert (os.path.isdir(cwd))
|
|
|
|
|
|
def HashLocalFile(path):
|
|
if not os.path.exists(path):
|
|
return None
|
|
with open(path, 'rb') as f:
|
|
return hashlib.sha256(f.read()).hexdigest()
|
|
|
|
|
|
def ExtractZipfilePreservePermissions(zf, info, path):
|
|
target_path = os.path.join(path, info.filename)
|
|
mode = info.external_attr >> 16
|
|
S_IFLNK = 0o120000 # symbolic link
|
|
if (mode & S_IFLNK) == S_IFLNK:
|
|
dst = zf.read(info).decode()
|
|
os.symlink(dst, target_path)
|
|
return
|
|
zf.extract(info.filename, path=path)
|
|
min_acls = 0o755 if info.filename.endswith('/') else 0o644
|
|
os.chmod(target_path, mode | min_acls)
|
|
|
|
|
|
def IsGitRepoCheckoutOutAtRevision(path, revision):
|
|
return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
|
|
|
|
|
|
def RmtreeIfExists(path):
|
|
# Git creates read-only files on windows, which cause failures with rmtree.
|
|
# This seems the socially accepted way to deal with it.
|
|
# See https://bugs.python.org/issue19643 .
|
|
def del_read_only_for_windows(_action, name, _exc):
|
|
os.chmod(name, stat.S_IWRITE)
|
|
os.remove(name)
|
|
|
|
if not os.path.exists(path):
|
|
return
|
|
third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
|
|
buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
|
|
test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
|
|
if (not os.path.abspath(path).startswith(buildtools_path) and
|
|
not os.path.abspath(path).startswith(test_path) and
|
|
not os.path.abspath(path).startswith(third_party_path)):
|
|
# Safety check to prevent that some merge confilct ends up doing some
|
|
# rm -rf / or similar.
|
|
logging.fatal(
|
|
'Cannot remove %s: outside of {buildtools, test/data, third_party}',
|
|
path)
|
|
sys.exit(1)
|
|
logging.info('Removing %s' % path)
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path, onerror=del_read_only_for_windows)
|
|
else:
|
|
os.remove(path)
|
|
|
|
|
|
def CheckoutGitRepo(path, git_url, revision, check_only):
|
|
if IsGitRepoCheckoutOutAtRevision(path, revision):
|
|
return False
|
|
if check_only:
|
|
return True
|
|
path = path.replace('/', os.sep)
|
|
RmtreeIfExists(path)
|
|
MkdirRecursive(path)
|
|
logging.info('Fetching %s @ %s into %s', git_url, revision, path)
|
|
subprocess.check_call(['git', 'init', path], cwd=path)
|
|
CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
|
|
cwd=path)
|
|
subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
|
|
CheckCallRetry(
|
|
['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
|
|
cwd=path)
|
|
assert (IsGitRepoCheckoutOutAtRevision(path, revision))
|
|
return True
|
|
|
|
|
|
def InstallNodeModules(force_clean=False):
|
|
if force_clean:
|
|
node_modules = os.path.join(UI_DIR, 'node_modules')
|
|
logging.info('Clearing %s', node_modules)
|
|
subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
|
|
cwd=ROOT_DIR)
|
|
logging.info("Running `pnpm install --shamefully-hoist --frozen-lockfile` in {0}".format(UI_DIR))
|
|
|
|
# Some node modules have postinstall scripts (already bad) but worse
|
|
# sometimes they are in the form: "postinstall: 'node ./scripts/foo'"
|
|
# so here we need to ensure that our hermetic node is available in
|
|
# PATH.
|
|
env = os.environ.copy()
|
|
env['PATH'] = TOOLS_DIR + ':' + env['PATH']
|
|
|
|
subprocess.check_call([
|
|
os.path.join(TOOLS_DIR, 'pnpm'),
|
|
'install',
|
|
'--shamefully-hoist',
|
|
'--frozen-lockfile'],
|
|
cwd=UI_DIR,
|
|
env=env)
|
|
# pbjs has the bad habit of installing extra packages on its first
|
|
# run. Run it here, so we avoid fetches while building.
|
|
pbjs = ['node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
|
|
subprocess.call(pbjs, cwd=UI_DIR, env=env)
|
|
with open(NODE_MODULES_STATUS_FILE, 'w') as f:
|
|
f.write(HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml')))
|
|
|
|
|
|
def CheckNodeModules():
|
|
"""Returns True if the modules are up-to-date.
|
|
|
|
There doesn't seem to be an easy way to check node modules versions. Instead
|
|
just check if pnpm-lock.json changed since the last `pnpm install` call.
|
|
"""
|
|
if not os.path.exists(NODE_MODULES_STATUS_FILE):
|
|
return False
|
|
with open(NODE_MODULES_STATUS_FILE, 'r') as f:
|
|
actual = f.read()
|
|
expected = HashLocalFile(os.path.join(UI_DIR, 'pnpm-lock.yaml'))
|
|
return expected == actual
|
|
|
|
|
|
def CheckHashes():
|
|
for dep in ALL_DEPS:
|
|
if dep.source_url.endswith('.git'):
|
|
continue
|
|
logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
|
|
dep.target_arch)
|
|
with tempfile.NamedTemporaryFile(delete=False) as f:
|
|
f.close()
|
|
DownloadURL(dep.source_url, f.name)
|
|
actual_checksum = HashLocalFile(f.name)
|
|
os.unlink(f.name)
|
|
if (actual_checksum != dep.checksum):
|
|
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
|
|
dep.source_url, dep.checksum, actual_checksum))
|
|
|
|
|
|
def CheckDepotToolsIsRecent():
|
|
gn_py_path = shutil.which('gn.py')
|
|
if gn_py_path is None:
|
|
return True # depot_tools doesn't seem to be installed in the PATH.
|
|
dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
|
|
cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
|
|
git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
|
|
if git_ret == 0:
|
|
return True
|
|
print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
|
|
print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
|
|
(dt_dir, dt_dir))
|
|
return False
|
|
|
|
|
|
def Main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--android',
|
|
action='store_true',
|
|
help='NDK and emulator images target_os="android"')
|
|
parser.add_argument(
|
|
'--linux-arm',
|
|
action='store_true',
|
|
help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
|
|
parser.add_argument(
|
|
'--ui',
|
|
action='store_true',
|
|
help='Node and NPM packages to Build the Web-based UI via ./ui/build')
|
|
parser.add_argument(
|
|
'--grpc', action='store_true', help='Packages to build gRPC')
|
|
parser.add_argument('--check-only')
|
|
parser.add_argument('--filter', action='append')
|
|
parser.add_argument('--verify', help='Check all URLs', action='store_true')
|
|
parser.add_argument(
|
|
'--no-toolchain', help='Do not download toolchain', action='store_true')
|
|
parser.add_argument(
|
|
'--build-os',
|
|
default=system().lower(),
|
|
choices=['windows', 'darwin', 'linux'],
|
|
help='Override the autodetected build operating system')
|
|
parser.add_argument(
|
|
'--build-arch',
|
|
default=GetArch(),
|
|
choices=['arm64', 'x64'],
|
|
help='Override the autodetected build CPU architecture')
|
|
args = parser.parse_args()
|
|
if args.verify:
|
|
CheckHashes()
|
|
return 0
|
|
|
|
target_os = args.build_os
|
|
if args.ui and target_os == 'windows':
|
|
print('Building the UI on Windows is unsupported')
|
|
return 1
|
|
|
|
if not CheckDepotToolsIsRecent():
|
|
return 1
|
|
|
|
deps = BUILD_DEPS_HOST
|
|
if not args.no_toolchain:
|
|
deps += BUILD_DEPS_TOOLCHAIN_HOST
|
|
if args.android:
|
|
deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
|
|
if args.linux_arm:
|
|
deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
|
|
if args.ui:
|
|
deps += UI_DEPS
|
|
if args.grpc:
|
|
deps += GRPC_DEPS
|
|
deps_updated = False
|
|
nodejs_updated = False
|
|
|
|
for old_dir in CLEANUP_OLD_DIRS:
|
|
RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
|
|
|
|
for dep in deps:
|
|
target_arch = args.build_arch
|
|
matches_os = dep.target_os == 'all' or target_os == dep.target_os
|
|
matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
|
|
if not matches_os or not matches_arch:
|
|
continue
|
|
if args.filter and not any(f in dep.target_folder for f in args.filter):
|
|
continue
|
|
local_path = os.path.join(ROOT_DIR, dep.target_folder)
|
|
if dep.source_url.endswith('.git'):
|
|
deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
|
|
args.check_only)
|
|
continue
|
|
is_compressed = any([local_path.endswith(ext) for ext in ['.zip', '.tgz', '.tbz2']])
|
|
compressed_target_dir = os.path.splitext(local_path)[0] if is_compressed else None
|
|
compressed_dir_stamp = os.path.join(compressed_target_dir, '.stamp') if is_compressed else None
|
|
|
|
if ((not is_compressed and HashLocalFile(local_path) == dep.checksum) or
|
|
(is_compressed and ReadFile(compressed_dir_stamp) == dep.checksum)):
|
|
continue
|
|
deps_updated = True
|
|
if args.check_only:
|
|
continue
|
|
MkdirRecursive(os.path.dirname(dep.target_folder))
|
|
if HashLocalFile(local_path) != dep.checksum:
|
|
download_path = local_path + '.tmp'
|
|
logging.info('Downloading %s from %s', local_path, dep.source_url)
|
|
DownloadURL(dep.source_url, download_path)
|
|
os.chmod(download_path, 0o755)
|
|
actual_checksum = HashLocalFile(download_path)
|
|
if (actual_checksum != dep.checksum):
|
|
os.remove(download_path)
|
|
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
|
|
download_path, dep.checksum, actual_checksum))
|
|
return 1
|
|
shutil.move(download_path, local_path)
|
|
if 'nodejs' in dep.target_folder:
|
|
nodejs_updated = True
|
|
|
|
assert (HashLocalFile(local_path) == dep.checksum)
|
|
|
|
if is_compressed:
|
|
logging.info('Extracting %s into %s' % (local_path, compressed_target_dir))
|
|
assert (os.path.commonprefix((ROOT_DIR, compressed_target_dir)) == ROOT_DIR)
|
|
RmtreeIfExists(compressed_target_dir)
|
|
|
|
# Decompress the archive.
|
|
if local_path.endswith('.tgz'):
|
|
MkdirRecursive(compressed_target_dir)
|
|
subprocess.check_call(['tar', '-oxf', local_path], cwd=compressed_target_dir)
|
|
elif local_path.endswith('.zip'):
|
|
with zipfile.ZipFile(local_path, 'r') as zf:
|
|
for info in zf.infolist():
|
|
ExtractZipfilePreservePermissions(zf, info, compressed_target_dir)
|
|
elif local_path.endswith('.tbz2'):
|
|
tar_path = '{}.tar.tmp'.format(local_path)
|
|
with open(tar_path, 'w') as f:
|
|
with bz2.open(local_path, 'r') as bf:
|
|
f.write(bf.read())
|
|
MkdirRecursive(compressed_target_dir)
|
|
subprocess.check_call(['tar', '-oxf', tar_path], cwd=compressed_target_dir)
|
|
|
|
# If the zip contains one root folder, rebase one level up moving all
|
|
# its sub files and folders inside |target_dir|.
|
|
subdir = os.listdir(compressed_target_dir)
|
|
if len(subdir) == 1:
|
|
subdir = os.path.join(compressed_target_dir, subdir[0])
|
|
if os.path.isdir(subdir):
|
|
for subf in os.listdir(subdir):
|
|
shutil.move(os.path.join(subdir, subf), compressed_target_dir)
|
|
os.rmdir(subdir)
|
|
|
|
# Create stamp and remove the archive.
|
|
with open(compressed_dir_stamp, 'w') as stamp_file:
|
|
stamp_file.write(dep.checksum)
|
|
os.remove(local_path)
|
|
|
|
if args.ui:
|
|
# Needs to happen after nodejs is installed above.
|
|
if args.check_only:
|
|
deps_updated |= not CheckNodeModules()
|
|
else:
|
|
InstallNodeModules(force_clean=nodejs_updated)
|
|
|
|
cur_python_interpreter = sys.executable
|
|
test_data_synced = 0 == subprocess.call([
|
|
cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
|
|
'--ignore-new'
|
|
])
|
|
if args.check_only:
|
|
if not deps_updated and test_data_synced:
|
|
with open(args.check_only, 'w') as f:
|
|
f.write('OK') # The content is irrelevant, just keep GN happy.
|
|
return 0
|
|
argz = ' '.join(
|
|
[x for x in sys.argv[1:] if not x.startswith('--check-only')])
|
|
print('\033[91mBuild deps are stale. ' +
|
|
'Please run tools/install-build-deps %s\033[0m' % argz)
|
|
if not test_data_synced:
|
|
print('//test/data/ is out of sync. `tools/test_data status` for details')
|
|
return 1
|
|
|
|
if not test_data_synced:
|
|
cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
|
|
if not sys.stdout.isatty():
|
|
cmd += ['--verbose'] # For CI bots
|
|
subprocess.check_call(cmd)
|
|
|
|
if deps_updated:
|
|
# Stale binary files may be compiled against old sysroot headers that aren't
|
|
# tracked by gn.
|
|
logging.warning('Remember to run "gn clean <output_directory>" ' +
|
|
'to avoid stale binary files.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logging.basicConfig(level=logging.INFO)
|
|
sys.exit(Main())
|