Move the CI prep from repeated lines in the YML to a shell script

Oh, this is much easier to work with.

We can now detect the version of PostgreSQL installed and shouldn't
need to update CI just because PostgreSQL has been updated in Debian
Unstable.
This commit is contained in:
Andrew Ruthven 2024-02-18 13:21:01 +13:00
parent 4ae36c132d
commit d7d7dd2ade
3 changed files with 268 additions and 221 deletions

View File

@ -44,49 +44,10 @@ test:
when:
always
script:
- apt-get -y update
- bash -c 'mkdir -p /usr/share/man/man{0..10}'
- apt-get -y install locales
- echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
- locale-gen
- echo "LANG=en_NZ.UTF-8" > /etc/default/locale
- apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet
- curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf -
- mv awl-master /usr/share/awl/
- chown -R www-data /usr/share/awl/
- dpkg --ignore-depends=libawl-php -i *.deb
- echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts
- rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf
- cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf
- sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf
- mkdir /usr/share/davical/testing/
- cp testing/*.php /usr/share/davical/testing/
- mkdir /usr/share/davical/htdocs/testfiles
- ln -s /usr/share/davical/htdocs/testfiles htdocs
- rm /etc/davical/config.php
- cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php
- mkdir -p /var/log/davical
- chown www-data /var/log/davical
- sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf
- echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf
- pg_ctlcluster 16 main start
- su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
- su postgres -c 'createuser davical_app --superuser'
- su postgres -c 'createuser testrunner --superuser'
- pg_ctlcluster 16 main restart
- a2enmod rewrite
- a2enmod headers
- apache2ctl start
- useradd testrunner
- testing/gitlab_ci_script.sh
- cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x'
after_script:
- mkdir -p apache2_log
- cp -r /var/log/apache2 apache2_log/build
- xz apache2_log/build/*
- mkdir -p davical_log
- cp -r /var/log/davical davical_log/build
- testing/gitlab_ci_after_script.sh build
test_bullseye_carddavclientinterop:
@ -106,54 +67,12 @@ test_bullseye_carddavclientinterop:
when:
always
script:
- apt-get -y update
- bash -c 'mkdir -p /usr/share/man/man{0..10}'
- apt-get -y install locales
- echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
- locale-gen
- echo "LANG=en_NZ.UTF-8" > /etc/default/locale
- apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml postgresql-client postgresql libapache2-mod-php curl xmlstarlet composer phpunit
- curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf -
- mv awl-master /usr/share/awl/
- chown -R www-data /usr/share/awl/
- dpkg --ignore-depends=libawl-php -i *.deb
- echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts
- rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf
- cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf
- sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf
- mkdir /usr/share/davical/testing/
- cp testing/*.php /usr/share/davical/testing/
- rm /etc/davical/config.php
- cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php
- mkdir -p /var/log/davical
- chown www-data /var/log/davical
- sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf
- echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf
- pg_ctlcluster 13 main start
- su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
- su postgres -c 'createuser davical_app --superuser'
- su postgres -c 'createuser testrunner --superuser'
- pg_ctlcluster 13 main restart
- a2enmod rewrite
- a2enmod headers
- apache2ctl start
- useradd testrunner
- cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x'
- cd ..
- curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf -
- cd carddavclient-master
- composer install
- echo '<?php declare(strict_types=1); namespace MStilkerich\Tests\CardDavClient\Interop; use MStilkerich\CardDavClient\{Account,AddressbookCollection,Config}; final class AccountData { public const ACCOUNTS = [ "Davical" => [ "username" => "user3", "password" => "user3", "discoveryUri" => "http://regression", "syncAllowExtraChanges" => false, "featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL, ], ]; public const ADDRESSBOOKS = [ "Davical_0" => [ "account" => "Davical", "url" => "http://regression/caldav.php/user3/addresses/", "displayname" => "user3 addresses", "description" => null, ], ]; }' > tests/Interop/AccountData.php
- mkdir -p testreports/interop
- vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage
- testing/gitlab_ci_script.sh interop
# Needed to create the regression database. Could be replaced by a basic suite.
- cd testing && su testrunner -c 'IS_CI=yes ./run_regressions.sh regression-suite x'
- cd ../carddavclient-master && vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage
after_script:
- mkdir -p apache2_log
- cp -r /var/log/apache2 apache2_log/test_bullseye_carddavclientinterop
- bzip2 apache2_log/test_bullseye_carddavclientinterop/*
- mkdir -p davical_log
- cp -r /var/log/davical davical_log/test_bullseye_carddavclientinterop
- testing/gitlab_ci_after_script.sh
build_bullseye_latestphp:
@ -198,53 +117,10 @@ test_bullseye_latestphp:
when:
always
script:
- apt-get -y update
- bash -c 'mkdir -p /usr/share/man/man{0..10}'
- apt-get -y install locales
- echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
- locale-gen
- echo "LANG=en_NZ.UTF-8" > /etc/default/locale
- apt-get -y install libdbd-pg-perl libyaml-perl perl postgresql postgresql-client libpq-dev xmlstarlet
- curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf -
- mv awl-master /usr/share/awl/
- chown -R www-data /usr/share/awl/
- dpkg --ignore-depends=php,php-pgsql,php-xml,libawl-php,php-cli -i *.deb
- docker-php-ext-install -j$(nproc) pgsql
- docker-php-ext-install -j$(nproc) pdo_pgsql
- docker-php-ext-install -j$(nproc) calendar
- echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts
- rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf
- cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf
- sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf
- mkdir /usr/share/davical/testing/
- cp testing/*.php /usr/share/davical/testing/
- mkdir /usr/share/davical/htdocs/testfiles
- ln -s /usr/share/davical/htdocs/testfiles htdocs
- rm /etc/davical/config.php
- cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php
- mkdir -p /var/log/davical
- chown www-data /var/log/davical
- sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf
- echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf
- pg_ctlcluster 13 main start
- su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
- su postgres -c 'createuser davical_app --superuser'
- su postgres -c 'createuser testrunner --superuser'
- pg_ctlcluster 13 main restart
- a2enmod rewrite
- a2enmod headers
- a2dismod -f deflate
- apache2ctl start
- useradd testrunner
- testing/gitlab_ci_script.sh
- cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x'
after_script:
- mkdir -p apache2_log
- cp -r /var/log/apache2 apache2_log/test_bullseye_latestphp
- bzip2 apache2_log/test_bullseye_latestphp/* || true
- mkdir -p davical_log
- cp -r /var/log/davical davical_log/test_bullseye_latestphp
- testing/gitlab_ci_after_script.sh
test_memcache:
stage: testall
@ -259,53 +135,10 @@ test_memcache:
when:
always
script:
- apt-get -y update
- bash -c 'mkdir -p /usr/share/man/man{0..10}'
- apt-get -y install locales
- echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
- locale-gen
- echo "LANG=en_NZ.UTF-8" > /etc/default/locale
- apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-memcached php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet memcached netcat-openbsd
- phpenmod memcached
- curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf -
- mv awl-master /usr/share/awl/
- chown -R www-data /usr/share/awl/
- dpkg --ignore-depends=libawl-php -i *.deb
- echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts
- rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf
- cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf
- sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf
- mkdir /usr/share/davical/testing/
- cp testing/*.php /usr/share/davical/testing/
- mkdir /usr/share/davical/htdocs/testfiles
- ln -s /usr/share/davical/htdocs/testfiles htdocs
- rm /etc/davical/config.php
- cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed 's.//memcache ..g' > /etc/davical/regression-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php
- mkdir -p /var/log/davical
- chown www-data /var/log/davical
- sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf
- echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf
- pg_ctlcluster 16 main start
- su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
- su postgres -c 'createuser davical_app --superuser'
- su postgres -c 'createuser testrunner --superuser'
- pg_ctlcluster 16 main restart
- a2enmod rewrite
- a2enmod headers
- apache2ctl start
- /etc/init.d/memcached start
- useradd testrunner
# testrunner needs to be able to read /var/log/apache2/regression-error.log for the memcache tests.
- adduser testrunner adm
- testing/gitlab_ci_script.sh memcache
- cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x'
after_script:
- mkdir -p apache2_log
- cp -r /var/log/apache2 apache2_log/test_memcache
- xz apache2_log/test_memcache/*
- mkdir -p davical_log
- cp -r /var/log/davical davical_log/test_memcache
- testing/gitlab_ci_after_script.sh
test_ldap:
stage: testldap
@ -320,48 +153,7 @@ test_ldap:
when:
always
script:
- apt-get -y update
- bash -c 'mkdir -p /usr/share/man/man{0..10}'
- apt-get -y install locales
- echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
- locale-gen
- echo "LANG=en_NZ.UTF-8" > /etc/default/locale
- apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-ldap php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet libnet-ldap-server-test-perl netcat-openbsd
- phpenmod ldap
- curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf -
- mv awl-master /usr/share/awl/
- chown -R www-data /usr/share/awl/
- dpkg --ignore-depends=libawl-php -i *.deb
- echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts
- rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf
- cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf
- sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf
- mkdir /usr/share/davical/testing/
- cp testing/*.php /usr/share/davical/testing/
- mkdir /usr/share/davical/htdocs/testfiles
- ln -s /usr/share/davical/htdocs/testfiles htdocs
- rm /etc/davical/config.php
- cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed -E 's.//(memcache|ldap) ..g' > /etc/davical/regression-conf.php
- ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav_ldap-conf.php
- mkdir -p /var/log/davical
- chown www-data /var/log/davical
- sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf
- echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf
- pg_ctlcluster 16 main start
- su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
- su postgres -c 'createuser davical_app --superuser'
- su postgres -c 'createuser testrunner --superuser'
- pg_ctlcluster 16 main restart
- a2enmod rewrite
- a2enmod headers
- apache2ctl start
- useradd testrunner
# testrunner needs to be able to read /var/log/apache2/regression-error.log for the ldap tests.
- adduser testrunner adm
- testing/gitlab_ci_script.sh ldap
- cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="ldap" ./run_regressions.sh all x'
after_script:
- mkdir -p apache2_log
- cp -r /var/log/apache2 apache2_log/test_ldap
- xz apache2_log/test_memcache_and_ldap/*
- mkdir -p davical_log
- cp -r /var/log/davical davical_log/test_ldap
- testing/gitlab_ci_after_script.sh

View File

@ -0,0 +1,16 @@
#!/bin/bash
# Order: commandline, $CI_JOB_NAME, unknown
test="${1:-$CI_JOB_NAME}"
test="${test:-unknown}"
if [ -d /var/log/apache2 ]; then
mkdir -p apache2_log
cp -r /var/log/apache2 "apache2_log/$test"
xz apache2_log/"$test"/*
fi
if [ -d /var/log/davical ]; then
mkdir -p davical_log
cp -r /var/log/davical "davical_log/$test"
fi

239
testing/gitlab_ci_script.sh Executable file
View File

@ -0,0 +1,239 @@
#!/bin/bash
# Configure the virtual environment that Gitlab provides for our CI jobs.
set -e
set -x
mode=${1:-$CI_JOB_NAME}
ldap=0
memcache=0
interop=0
if [[ $mode =~ "ldap" ]]; then
ldap=1
memcache=1
elif [[ $mode =~ "memcache" ]]; then
memcache=1
elif [[ $mode =~ "interop" ]]; then
interop=1
fi
# The images don't have man page directories?
mkdir -p /usr/share/man/man{0..10}
# When we install the davical deb, ignore libawl-php version dependency, as
# we also install the libawl-php .deb built by the AWL CI.
dpkg_ignore_depends="libawl-php"
# Images have no package info, load it.
apt-get -y update
###
### Pre-configure the locale so it is ready before other packages are
### installed.
###
apt-get -y install locales
echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen
locale-gen
echo "LANG=en_NZ.UTF-8" > /etc/default/locale
###
### Install packages required for testing
###
packages="libdbd-pg-perl libyaml-perl postgresql-client postgresql
curl xmlstarlet netcat-openbsd"
if [[ $mode =~ "latestphp" ]]; then
# PHP pgsql package is built from source, needs libpq-dev.
packages="$packages libpq-dev"
else
# The latestphp images already have some of these available, and others need
# to installed in another method.
packages="$packages libapache2-mod-php php php-cli php-pgsql php-xml
php-curl"
fi
if [ $memcache == 1 ]; then
packages="$packages php-memcached memcached"
fi
if [ $ldap == 1 ]; then
packages="$packages php-ldap libnet-ldap-server-test-perl"
fi
if [ $interop == 1 ]; then
packages="$packages composer phpunit"
fi
apt-get -y install $packages
###
### Setup PHP for we're using latestphp...
###
if [[ $mode =~ "latestphp" ]]; then
# The image doesn't install debs, so ignore the dependencies.
dpkg_ignore_depends="$dpkg_ignore_depends,php,php-pgsql,php-xml,php-cli"
docker-php-ext-install -j$(nproc) pgsql
docker-php-ext-install -j$(nproc) pdo_pgsql
docker-php-ext-install -j$(nproc) calendar
# The PHP 8.2 container has deflate enabled in Apache2, disable it.
# This is causes a test to fail because we're being sent a gzip
# compressed result. My reading of the curl manpage says that it should
# be decompressed by curl, but it isn't. Let's just disable the deflate
# module.
a2dismod -f deflate
fi
###
### Setup memcache if we're using it.
###
if [ $memcache == 1 ]; then
phpenmod memcached
/etc/init.d/memcached start
conf_filter_memcache="| sed 's.//memcache ..g'"
fi
###
### Prepare PHP for LDAP if we're using it.
###
if [ $ldap == 1 ]; then
phpenmod ldap
conf_filter_ldap="| sed 's.//ldap ..g'"
fi
###
### Install the latest build of AWL
###
curl https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz \
| tar zxf -
mv awl-master /usr/share/awl/
chown -R www-data /usr/share/awl/
###
### Install the davical and libawl-php packages to test
###
dpkg --ignore-depends=$dpkg_ignore_depends -i *.deb
mkdir /usr/share/davical/testing/
cp testing/*.php /usr/share/davical/testing/
mkdir -p /var/log/davical
chown www-data /var/log/davical
###
### Setup DAViCal config files
###
rm /etc/davical/config.php
eval cat testing/regression-conf.php.example \
\| sed 's.//\$c-\>dbg.\$c-\>dbg.' \
$conf_filter_memcache \
$conf_filter_ldap \
> /etc/davical/regression-conf.php
for site in mycaldav mycaldav_ldap myempty; do
ln -s /etc/davical/regression-conf.php /etc/davical/$site-conf.php
done
###
### Setup PostgreSQL
###
pg_ver=$(ls /etc/postgresql)
sed -i '/peer/d' /etc/postgresql/$pg_ver/main/pg_hba.conf
echo 'local all all trust' >> /etc/postgresql/$pg_ver/main/pg_hba.conf
pg_ctlcluster $pg_ver main start
su postgres -c 'createuser davical_dba --createdb --createrole --superuser'
su postgres -c 'createuser davical_app --superuser'
su postgres -c 'createuser testrunner --superuser'
# Why? You don't need to restart PG after creating users.
#pg_ctlcluster $pg_ver main restart
###
### Setup testrunner user
###
useradd testrunner
# testrunner needs to be able to read /var/log/apache2/regression-error.log
# for the memcache tests.
adduser testrunner adm
###
### Configure Apache2
###
echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts
rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf \
&& touch /etc/apache2/ports.conf
cp testing/apache-site.conf.example \
/etc/apache2/sites-enabled/davical-regression.conf
sed -i 's/\/path\/to/\/usr\/share/g' \
/etc/apache2/sites-enabled/davical-regression.conf
a2enmod rewrite
a2enmod headers
apache2ctl start
# Ensure that the jobs that need to drop in static files for Apache to serve
# up can do so.
mkdir /usr/share/davical/htdocs/testfiles
chown testrunner /usr/share/davical/htdocs/testfiles
ln -s /usr/share/davical/htdocs/testfiles htdocs
###
### Set up for carddavclientinterop
###
if [ $interop == 1 ]; then
curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf -
cd carddavclient-master
composer install
cat <<EOF > tests/Interop/AccountData.php
<?php
declare(strict_types=1);
namespace MStilkerich\Tests\CardDavClient\Interop;
use MStilkerich\CardDavClient\Account;
use MStilkerich\CardDavClient\AddressbookCollection;
use MStilkerich\CardDavClient\Config;
final class AccountData {
public const ACCOUNTS = [
"Davical" => [
"username" => "user3",
"password" => "user3",
"discoveryUri" => "http://regression",
"featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL,
"syncAllowExtraChanges" => false,
],
];
public const ADDRESSBOOKS = [
"Davical_0" => [
"account" => "Davical",
"url" => "http://regression/caldav.php/user3/addresses/",
"displayname" => "user3 addresses",
"description" => null,
],
];
}
EOF
mkdir -p testreports/interop
fi