From d7d7dd2ade3f16d84c455bacf6173f8daf3540c8 Mon Sep 17 00:00:00 2001 From: Andrew Ruthven Date: Sun, 18 Feb 2024 13:21:01 +1300 Subject: [PATCH] Move the CI prep from repeated lines in the YML to a shell script Oh, this is much easier to work with. We can now detect the version of PostgreSQL installed and shouldn't need to update CI just because PostgreSQL has been updated in Debian Unstable. --- .gitlab-ci.yml | 234 ++--------------------------- testing/gitlab_ci_after_script.sh | 16 ++ testing/gitlab_ci_script.sh | 239 ++++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 221 deletions(-) create mode 100755 testing/gitlab_ci_after_script.sh create mode 100755 testing/gitlab_ci_script.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f665ebb8..4bd97819 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,49 +44,10 @@ test: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner + - testing/gitlab_ci_script.sh - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/build - - xz apache2_log/build/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/build + - testing/gitlab_ci_after_script.sh build test_bullseye_carddavclientinterop: @@ -106,54 +67,12 @@ test_bullseye_carddavclientinterop: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml postgresql-client postgresql libapache2-mod-php curl xmlstarlet composer phpunit - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf - - pg_ctlcluster 13 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 13 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner - - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' - - cd .. - - curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf - - - cd carddavclient-master - - composer install - - echo ' [ "username" => "user3", "password" => "user3", "discoveryUri" => "http://regression", "syncAllowExtraChanges" => false, "featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL, ], ]; public const ADDRESSBOOKS = [ "Davical_0" => [ "account" => "Davical", "url" => "http://regression/caldav.php/user3/addresses/", "displayname" => "user3 addresses", "description" => null, ], ]; }' > tests/Interop/AccountData.php - - mkdir -p testreports/interop - - vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage + - testing/gitlab_ci_script.sh interop + # Needed to create the regression database. Could be replaced by a basic suite. + - cd testing && su testrunner -c 'IS_CI=yes ./run_regressions.sh regression-suite x' + - cd ../carddavclient-master && vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_bullseye_carddavclientinterop - - bzip2 apache2_log/test_bullseye_carddavclientinterop/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_bullseye_carddavclientinterop + - testing/gitlab_ci_after_script.sh build_bullseye_latestphp: @@ -198,53 +117,10 @@ test_bullseye_latestphp: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl perl postgresql postgresql-client libpq-dev xmlstarlet - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=php,php-pgsql,php-xml,libawl-php,php-cli -i *.deb - - docker-php-ext-install -j$(nproc) pgsql - - docker-php-ext-install -j$(nproc) pdo_pgsql - - docker-php-ext-install -j$(nproc) calendar - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf - - pg_ctlcluster 13 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 13 main restart - - a2enmod rewrite - - a2enmod headers - - a2dismod -f deflate - - apache2ctl start - - useradd testrunner + - testing/gitlab_ci_script.sh - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_bullseye_latestphp - - bzip2 apache2_log/test_bullseye_latestphp/* || true - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_bullseye_latestphp + - testing/gitlab_ci_after_script.sh test_memcache: stage: testall @@ -259,53 +135,10 @@ test_memcache: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-memcached php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet memcached netcat-openbsd - - phpenmod memcached - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed 's.//memcache ..g' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - /etc/init.d/memcached start - - useradd testrunner - # testrunner needs to be able to read /var/log/apache2/regression-error.log for the memcache tests. - - adduser testrunner adm + - testing/gitlab_ci_script.sh memcache - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_memcache - - xz apache2_log/test_memcache/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_memcache + - testing/gitlab_ci_after_script.sh test_ldap: stage: testldap @@ -320,48 +153,7 @@ test_ldap: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-ldap php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet libnet-ldap-server-test-perl netcat-openbsd - - phpenmod ldap - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed -E 's.//(memcache|ldap) ..g' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav_ldap-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner - # testrunner needs to be able to read /var/log/apache2/regression-error.log for the ldap tests. - - adduser testrunner adm + - testing/gitlab_ci_script.sh ldap - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="ldap" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_ldap - - xz apache2_log/test_memcache_and_ldap/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_ldap + - testing/gitlab_ci_after_script.sh diff --git a/testing/gitlab_ci_after_script.sh b/testing/gitlab_ci_after_script.sh new file mode 100755 index 00000000..b581432c --- /dev/null +++ b/testing/gitlab_ci_after_script.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Order: commandline, $CI_JOB_NAME, unknown +test="${1:-$CI_JOB_NAME}" +test="${test:-unknown}" + +if [ -d /var/log/apache2 ]; then + mkdir -p apache2_log + cp -r /var/log/apache2 "apache2_log/$test" + xz apache2_log/"$test"/* +fi + +if [ -d /var/log/davical ]; then + mkdir -p davical_log + cp -r /var/log/davical "davical_log/$test" +fi diff --git a/testing/gitlab_ci_script.sh b/testing/gitlab_ci_script.sh new file mode 100755 index 00000000..4a7fbbb7 --- /dev/null +++ b/testing/gitlab_ci_script.sh @@ -0,0 +1,239 @@ +#!/bin/bash + +# Configure the virtual environment that Gitlab provides for our CI jobs. + +set -e +set -x + +mode=${1:-$CI_JOB_NAME} + +ldap=0 +memcache=0 +interop=0 + +if [[ $mode =~ "ldap" ]]; then + ldap=1 + memcache=1 +elif [[ $mode =~ "memcache" ]]; then + memcache=1 +elif [[ $mode =~ "interop" ]]; then + interop=1 +fi + +# The images don't have man page directories? +mkdir -p /usr/share/man/man{0..10} + +# When we install the davical deb, ignore libawl-php version dependency, as +# we also install the libawl-php .deb built by the AWL CI. +dpkg_ignore_depends="libawl-php" + +# Images have no package info, load it. +apt-get -y update + +### +### Pre-configure the locale so it is ready before other packages are +### installed. +### + +apt-get -y install locales +echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen +locale-gen +echo "LANG=en_NZ.UTF-8" > /etc/default/locale + +### +### Install packages required for testing +### + +packages="libdbd-pg-perl libyaml-perl postgresql-client postgresql + curl xmlstarlet netcat-openbsd" + +if [[ $mode =~ "latestphp" ]]; then + # PHP pgsql package is built from source, needs libpq-dev. + packages="$packages libpq-dev" +else + # The latestphp images already have some of these available, and others need + # to installed in another method. + packages="$packages libapache2-mod-php php php-cli php-pgsql php-xml + php-curl" +fi + +if [ $memcache == 1 ]; then + packages="$packages php-memcached memcached" +fi + +if [ $ldap == 1 ]; then + packages="$packages php-ldap libnet-ldap-server-test-perl" +fi + +if [ $interop == 1 ]; then + packages="$packages composer phpunit" +fi + +apt-get -y install $packages + +### +### Setup PHP for we're using latestphp... +### + +if [[ $mode =~ "latestphp" ]]; then + # The image doesn't install debs, so ignore the dependencies. + dpkg_ignore_depends="$dpkg_ignore_depends,php,php-pgsql,php-xml,php-cli" + + docker-php-ext-install -j$(nproc) pgsql + docker-php-ext-install -j$(nproc) pdo_pgsql + docker-php-ext-install -j$(nproc) calendar + + # The PHP 8.2 container has deflate enabled in Apache2, disable it. + # This is causes a test to fail because we're being sent a gzip + # compressed result. My reading of the curl manpage says that it should + # be decompressed by curl, but it isn't. Let's just disable the deflate + # module. + a2dismod -f deflate +fi + +### +### Setup memcache if we're using it. +### + +if [ $memcache == 1 ]; then + phpenmod memcached + /etc/init.d/memcached start + + conf_filter_memcache="| sed 's.//memcache ..g'" +fi + +### +### Prepare PHP for LDAP if we're using it. +### + +if [ $ldap == 1 ]; then + phpenmod ldap + + conf_filter_ldap="| sed 's.//ldap ..g'" +fi + +### +### Install the latest build of AWL +### + +curl https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz \ + | tar zxf - +mv awl-master /usr/share/awl/ +chown -R www-data /usr/share/awl/ + +### +### Install the davical and libawl-php packages to test +### + +dpkg --ignore-depends=$dpkg_ignore_depends -i *.deb + +mkdir /usr/share/davical/testing/ +cp testing/*.php /usr/share/davical/testing/ +mkdir -p /var/log/davical +chown www-data /var/log/davical + +### +### Setup DAViCal config files +### + +rm /etc/davical/config.php +eval cat testing/regression-conf.php.example \ + \| sed 's.//\$c-\>dbg.\$c-\>dbg.' \ + $conf_filter_memcache \ + $conf_filter_ldap \ + > /etc/davical/regression-conf.php + +for site in mycaldav mycaldav_ldap myempty; do + ln -s /etc/davical/regression-conf.php /etc/davical/$site-conf.php +done + +### +### Setup PostgreSQL +### + +pg_ver=$(ls /etc/postgresql) +sed -i '/peer/d' /etc/postgresql/$pg_ver/main/pg_hba.conf +echo 'local all all trust' >> /etc/postgresql/$pg_ver/main/pg_hba.conf +pg_ctlcluster $pg_ver main start +su postgres -c 'createuser davical_dba --createdb --createrole --superuser' +su postgres -c 'createuser davical_app --superuser' +su postgres -c 'createuser testrunner --superuser' +# Why? You don't need to restart PG after creating users. +#pg_ctlcluster $pg_ver main restart + +### +### Setup testrunner user +### + +useradd testrunner +# testrunner needs to be able to read /var/log/apache2/regression-error.log +# for the memcache tests. +adduser testrunner adm + +### +### Configure Apache2 +### + +echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts + +rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf \ + && touch /etc/apache2/ports.conf + +cp testing/apache-site.conf.example \ + /etc/apache2/sites-enabled/davical-regression.conf +sed -i 's/\/path\/to/\/usr\/share/g' \ + /etc/apache2/sites-enabled/davical-regression.conf + +a2enmod rewrite +a2enmod headers +apache2ctl start + +# Ensure that the jobs that need to drop in static files for Apache to serve +# up can do so. +mkdir /usr/share/davical/htdocs/testfiles +chown testrunner /usr/share/davical/htdocs/testfiles +ln -s /usr/share/davical/htdocs/testfiles htdocs + +### +### Set up for carddavclientinterop +### + +if [ $interop == 1 ]; then + curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf - + + cd carddavclient-master + composer install + + cat < tests/Interop/AccountData.php + [ + "username" => "user3", + "password" => "user3", + "discoveryUri" => "http://regression", + "featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL, + "syncAllowExtraChanges" => false, + ], + ]; + + public const ADDRESSBOOKS = [ + "Davical_0" => [ + "account" => "Davical", + "url" => "http://regression/caldav.php/user3/addresses/", + "displayname" => "user3 addresses", + "description" => null, + ], + ]; +} +EOF + + mkdir -p testreports/interop +fi