Skip to content

Commit

Permalink
Create many tiny artifacts for faster download
Browse files Browse the repository at this point in the history 
And not escaping the constraints many systems have on /tmp size.
If someone installs multiple php versions at once, we still download the full package.

The full package also remains available for offline installations of multiple targets for now.

Signed-off-by: Bob Weinand <[email protected]>
  • Loading branch information
@bwoebi
bwoebi committed Nov 14, 2024
1 parent 1f33697 commit 37efa3b
Show file tree
Hide file tree
Showing 4 changed files with 152 additions and 35 deletions.
10 changes: 5 additions & 5 deletions .circleci/continue_config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3620,16 +3620,16 @@ jobs:
name: Setup docker container
shell: powershell.exe
command: |
mkdir extensions_windows_x86_64
mkdir extensions_windows_x86_64_debugsymbols
mkdir extensions_x86_64
mkdir extensions_x86_64_debugsymbols
# No DNS for you by default in circleci docker containers
docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net
docker run -v ${pwd}:C:\Users\ContainerAdministrator\app -d --network net --name php << parameters.docker_image >> ping -t localhost
- run:
name: Build nts
shell: powershell.exe
command: |
docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
docker exec php powershell.exe "cd app; switch-php nts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>.dll; move x64\Release\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>.pdb"
- run:
name: Reuse libdatadog build
shell: powershell.exe
Expand All @@ -3639,10 +3639,10 @@ jobs:
name: Build zts
shell: powershell.exe
command: |
docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_windows_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_windows_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
docker exec php powershell.exe "cd app; switch-php zts; C:\php\SDK\phpize.bat; .\configure.bat --enable-debug-pack; nmake; move x64\Release_TS\php_ddtrace.dll extensions_x86_64\php_ddtrace-<< parameters.so_suffix >>-zts.dll; move x64\Release_TS\php_ddtrace.pdb extensions_x86_64_debugsymbols\php_ddtrace-<< parameters.so_suffix >>-zts.pdb"
- persist_to_workspace:
root: '.'
paths: [ './extensions_windows_x86_64', './extensions_windows_x86_64_debugsymbols' ]
paths: [ './extensions_x86_64', './extensions_x86_64_debugsymbols' ]

compile_appsec_extension_centos:
working_directory: ~/datadog
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ build_pecl_package:
tooling/bin/pecl-build $${FILES//$${BUILD_DIR}/}

dbgsym.tar.gz:
$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_windows_x86_64_debugsymbols --owner=0 --group=0)
$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0)

installer_packages: .apk.x86_64 .apk.aarch64 .rpm.x86_64 .rpm.aarch64 .deb.x86_64 .deb.arm64 .tar.gz.x86_64 .tar.gz.aarch64 bundle.tar.gz dbgsym.tar.gz
tar --use-compress-program=pigz --exclude='dd-library-php-ssi-*' -cf packages.tar.gz $(PACKAGES_BUILD_DIR) --owner=0 --group=0
Expand Down
100 changes: 80 additions & 20 deletions datadog-setup.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,11 @@ function print_help()
option can be provided multiple times.
--install-dir <path> Install to a specific directory. Default: '$installdir'
--uninstall Uninstall the library from the specified binaries.
--file <path to .tar.gz> Path to a dd-library-php-*.tar.gz file. Can be used for offline installation.
--extension-dir <path> Specify the extension directory. Default: PHP's extension directory.
--ini <path> Specify the INI file to use. Default: <ini-dir>/98-ddtrace.ini
--enable-appsec Enable the application security monitoring module.
--enable-profiling Enable the profiling module.
--enable-profiling Enable the profiling module.
-d setting[=value] Used in conjunction with `config <set|get>`
command to specify the INI setting to get or set.
Expand Down Expand Up @@ -462,9 +463,58 @@ function install($options)
$selectedBinaries = require_binaries_or_exit($options);
$interactive = empty($options[OPT_PHP_BIN]);

$commandExtensionSuffixes = [];
$downloadVersions = [];
foreach ($selectedBinaries as $command => $fullPath) {
$binaryForLog = ($command === $fullPath) ? $fullPath : "$command ($fullPath)";
echo "Checking for binary: $binaryForLog\n";

check_php_ext_prerequisite_or_exit($fullPath, 'json');

$phpProperties = ini_values($fullPath);
if (!isset($phpProperties[INI_SCANDIR])) {
if (!isset($phpProperties[INI_MAIN])) {
if (IS_WINDOWS) {
$phpProperties[INI_MAIN] = dirname($fullPath) . "/php.ini";
} else {
print_error_and_exit(
"It is not possible to perform installation on this "
. "system because there is no scan directory and no "
. "configuration file loaded."
);
}
}

print_warning(
"Performing an installation without a scan directory may "
. "result in fragile installations that are broken by normal "
. "system upgrades. It is advisable to use the configure "
. "switch --with-config-file-scan-dir when building PHP."
);
}

// Suffix (zts/debug)
$extensionSuffix = '';
if (is_truthy($phpProperties[IS_DEBUG])) {
$extensionSuffix .= '-debug';
}
if (is_truthy($phpProperties[THREAD_SAFETY])) {
$extensionSuffix .= '-zts';
}

$commandExtensionSuffixes[$command] = $extensionSuffix;

$extensionVersion = $phpProperties[PHP_API];
$downloadVersions["$extensionVersion$extensionSuffix"] = true;
}

$tar_gz_suffix = "";
if (count($downloadVersions) === 1) {
$tar_gz_suffix = "-" . key($downloadVersions);
}

// Preparing clean tmp folder to extract files
$tmpDir = sys_get_temp_dir() . '/dd-install';
$tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}.tar.gz";
$tmpArchiveRoot = $tmpDir . '/dd-library-php';
$tmpArchiveTraceRoot = $tmpDir . '/dd-library-php/trace';
$tmpArchiveAppsecRoot = $tmpDir . '/dd-library-php/appsec';
Expand All @@ -490,8 +540,14 @@ function install($options)
print_warning('--' . OPT_FILE . ' option is intended for internal usage and can be removed without notice');
$tmpDirTarGz = $options[OPT_FILE];
} else {
$url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}.tar.gz";
download($url, $tmpDirTarGz);
for (;;) {
$url = RELEASE_URL_PREFIX . "dd-library-php-" . RELEASE_VERSION . "-{$platform}{$tar_gz_suffix}.tar.gz";
$tmpDirTarGz = $tmpDir . "/dd-library-php-{$platform}{$tar_gz_suffix}.tar.gz";
if (download($url, $tmpDirTarGz, $tar_gz_suffix != "")) {
break;
}
$tar_gz_suffix = ""; // retry with the full archive if the original download failed
}
}
if (!IS_WINDOWS || `where tar 2> nul` !== null) {
execute_or_exit(
Expand Down Expand Up @@ -574,15 +630,7 @@ function install($options)

// Copying the extension
$extensionVersion = $phpProperties[PHP_API];

// Suffix (zts/debug/alpine)
$extensionSuffix = '';
if (is_truthy($phpProperties[IS_DEBUG])) {
$extensionSuffix .= '-debug';
}
if (is_truthy($phpProperties[THREAD_SAFETY])) {
$extensionSuffix .= '-zts';
}
$extensionSuffix = $commandExtensionSuffixes[$command];

$extDir = isset($options[OPT_EXTENSION_DIR]) ? $options[OPT_EXTENSION_DIR] : $phpProperties[EXTENSION_DIR];
echo "Installing extension to $extDir\n";
Expand Down Expand Up @@ -1416,7 +1464,7 @@ function execute_or_exit($exitMessage, $command)
* @param string $url
* @param string $destination
*/
function download($url, $destination)
function download($url, $destination, $retry = false)
{
echo "Downloading installable archive from $url.\n";
echo "This operation might take a while.\n";
Expand Down Expand Up @@ -1444,12 +1492,18 @@ function download($url, $destination)
curl_setopt($ch, CURLOPT_NOPROGRESS, false);
$progress_counter = 0;
$return = curl_exec($ch);

$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($httpCode == 404) {
return false;
}

curl_close($ch);
fclose($fp);

if (false !== $return) {
echo $okMessage;
return;
return true;
}
// Otherwise we attempt other methods
}
Expand All @@ -1461,20 +1515,20 @@ function download($url, $destination)
if (!IS_WINDOWS && false !== exec('curl --version', $output, $statusCode) && $statusCode === 0) {
$curlInvocationStatusCode = 0;
system(
'curl -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
'curl -f -L --output ' . escapeshellarg($destination) . ' ' . escapeshellarg($url),
$curlInvocationStatusCode
);

if ($curlInvocationStatusCode === 0) {
echo $okMessage;
return;
return true;
}
// Otherwise we attempt other methods
}

// file_get_contents
if (is_truthy(ini_get('allow_url_fopen')) && extension_loaded('openssl')) {
ini_set("memory_limit", "1G"); // increase memory limit otherwise we may run OOM here.
ini_set("memory_limit", "2G"); // increase memory limit otherwise we may run OOM here.
$data = @file_get_contents($url);
// PHP doesn't like too long location headers, and on PHP 7.3 and older they weren't read at all.
// But this only really matters for CircleCI artifacts, so not too bad.
Expand All @@ -1491,11 +1545,14 @@ function download($url, $destination)
}
got_data: ;
if ($data == "" || false === file_put_contents($destination, $data)) {
if ($retry) {
return false;
}
print_error_and_exit("Error while downloading the installable archive from $url\n");
}

echo $okMessage;
return;
return true;

next_method:
}
Expand All @@ -1508,11 +1565,14 @@ function download($url, $destination)
);
if ($webRequestInvocationStatusCode === 0) {
echo $okMessage;
return;
return true;
}
// Otherwise we attempt other methods
}

if ($retry) {
return false;
}

echo "Error: Cannot download the installable archive.\n";
echo " One of the following prerequisites must be satisfied:\n";
Expand Down
75 changes: 66 additions & 9 deletions tooling/bin/generate-final-artifact.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,69 @@ tmp_folder_final=$tmp_folder/final

architectures=(x86_64 aarch64)

php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
php_apis+=(20151012 20160303 20170718 20180731)
fi

targets=(unknown-linux-gnu alpine-linux-musl)
if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
targets+=(windows)
fi

configs=("" -zts -debug -debug-zts)

ln_with_dir() {
mkdir -p $(dirname $2)
ln $1 $2
}

for architecture in "${architectures[@]}"; do
for php_api in "${php_apis[@]}"; do
for full_target in "${targets[@]}"; do
target=${full_target#*-}
alpine=$(if [[ $target == "linux-musl" ]]; then echo -alpine; fi)
ext=$([[ $target == "windows" ]] && echo dll || echo so)
for config in "${configs[@]}"; do
ddtrace_ext_path=./extensions_${architecture}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace-${php_api}${alpine}${config}.${ext}
if [[ -f ${ddtrace_ext_path} ]]; then
rm -rf $tmp_folder
mkdir -p $tmp_folder_final

trace_base_dir=${tmp_folder_final}/dd-library-php/trace
ln_with_dir ${ddtrace_ext_path} ${trace_base_dir}/ext/${php_api}/$(if [[ $target == "windows" ]]; then echo php_; fi)ddtrace${config}.${ext}
cp -r ./src ${trace_base_dir}

profiling_ext_path=./datadog-profiling/${architecture}-${full_target}/lib/php/${php_api}/datadog-profiling${config}.${ext}
if [[ -f ${profiling_ext_path} ]]; then
profiling_base_dir=${tmp_folder_final}/dd-library-php/profiling
ln_with_dir ${profiling_ext_path} ${profiling_base_dir}/ext/${php_api}/datadog-profiling${config}.${ext}

# Licenses
ln \
./profiling/LICENSE* \
./profiling/NOTICE \
${profiling_base_dir}/
fi

appsec_ext_path=./appsec_${architecture}/ddappsec-${php_api}${alpine}${config}.${ext}
if [[ -f ${appsec_ext_path} ]]; then
appsec_base_dir=${tmp_folder_final}/dd-library-php/appsec
ln_with_dir ${appsec_ext_path} ${appsec_base_dir}/ext/$php_api/ddappsec${config}.${ext}
ln_with_dir ./appsec_${architecture}/libddappsec-helper.so ${appsec_base_dir}/lib/libddappsec-helper.so
ln_with_dir ./appsec_${architecture}/recommended.json ${appsec_base_dir}/etc/recommended.json
fi

echo "$release_version" > ${tmp_folder_final}/dd-library-php/VERSION
tar -czv \
-f ${packages_build_dir}/dd-library-php-${release_version}-$architecture-$target-${php_api}${config}.tar.gz \
-C ${tmp_folder_final} . --owner=0 --group=0
fi
done
done
done
done

for architecture in "${architectures[@]}"; do
tmp_folder_final_gnu=$tmp_folder_final/$architecture-linux-gnu
tmp_folder_final_musl=$tmp_folder_final/$architecture-linux-musl
Expand All @@ -33,24 +96,18 @@ for architecture in "${architectures[@]}"; do
tmp_folder_final_musl_trace=$tmp_folder_final_musl/dd-library-php/trace
tmp_folder_final_windows_trace=$tmp_folder_final_windows/dd-library-php/trace

php_apis=(20190902 20200930 20210902 20220829 20230831 20240924)
if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
php_apis+=(20151012 20160303 20170718 20180731)
fi
for php_api in "${php_apis[@]}"; do
mkdir -p ${tmp_folder_final_gnu_trace}/ext/$php_api ${tmp_folder_final_musl_trace}/ext/$php_api;
if [[ -z ${DDTRACE_MAKE_PACKAGES_ASAN:-} ]]; then
ln ./extensions_${architecture}/ddtrace-$php_api.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace.so;
ln ./extensions_${architecture}/ddtrace-$php_api-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-zts.so;
ln ./extensions_${architecture}/ddtrace-$php_api-debug.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug.so;
ln ./extensions_${architecture}/ddtrace-$php_api-alpine.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace.so;
if [[ ${php_api} -ge 20151012 ]]; then # zts on alpine starting 7.0
ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
fi
ln ./extensions_${architecture}/ddtrace-$php_api-alpine-zts.so ${tmp_folder_final_musl_trace}/ext/$php_api/ddtrace-zts.so;
if [[ ${php_api} -ge 20170718 && $architecture == "x86_64" ]]; then # Windows support starts on 7.2
mkdir -p ${tmp_folder_final_windows_trace}/ext/$php_api;
ln ./extensions_windows_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
ln ./extensions_windows_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
ln ./extensions_${architecture}/php_ddtrace-$php_api.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace.dll;
ln ./extensions_${architecture}/php_ddtrace-$php_api-zts.dll ${tmp_folder_final_windows_trace}/ext/$php_api/php_ddtrace-zts.dll;
fi
else
ln ./extensions_${architecture}/ddtrace-$php_api-debug-zts.so ${tmp_folder_final_gnu_trace}/ext/$php_api/ddtrace-debug-zts.so;
Expand Down

0 comments on commit 37efa3b

Please sign in to comment.