diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f665ebb8..4bd97819 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,49 +44,10 @@ test: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner + - testing/gitlab_ci_script.sh - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/build - - xz apache2_log/build/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/build + - testing/gitlab_ci_after_script.sh build test_bullseye_carddavclientinterop: @@ -106,54 +67,12 @@ test_bullseye_carddavclientinterop: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml postgresql-client postgresql libapache2-mod-php curl xmlstarlet composer phpunit - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf - - pg_ctlcluster 13 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 13 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner - - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' - - cd .. - - curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf - - - cd carddavclient-master - - composer install - - echo ' [ "username" => "user3", "password" => "user3", "discoveryUri" => "http://regression", "syncAllowExtraChanges" => false, "featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL, ], ]; public const ADDRESSBOOKS = [ "Davical_0" => [ "account" => "Davical", "url" => "http://regression/caldav.php/user3/addresses/", "displayname" => "user3 addresses", "description" => null, ], ]; }' > tests/Interop/AccountData.php - - mkdir -p testreports/interop - - vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage + - testing/gitlab_ci_script.sh interop + # Needed to create the regression database. Could be replaced by a basic suite. + - cd testing && su testrunner -c 'IS_CI=yes ./run_regressions.sh regression-suite x' + - cd ../carddavclient-master && vendor/bin/phpunit -c tests/Interop/phpunit.xml --no-coverage after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_bullseye_carddavclientinterop - - bzip2 apache2_log/test_bullseye_carddavclientinterop/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_bullseye_carddavclientinterop + - testing/gitlab_ci_after_script.sh build_bullseye_latestphp: @@ -198,53 +117,10 @@ test_bullseye_latestphp: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl perl postgresql postgresql-client libpq-dev xmlstarlet - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=php,php-pgsql,php-xml,libawl-php,php-cli -i *.deb - - docker-php-ext-install -j$(nproc) pgsql - - docker-php-ext-install -j$(nproc) pdo_pgsql - - docker-php-ext-install -j$(nproc) calendar - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/13/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/13/main/pg_hba.conf - - pg_ctlcluster 13 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 13 main restart - - a2enmod rewrite - - a2enmod headers - - a2dismod -f deflate - - apache2ctl start - - useradd testrunner + - testing/gitlab_ci_script.sh - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_bullseye_latestphp - - bzip2 apache2_log/test_bullseye_latestphp/* || true - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_bullseye_latestphp + - testing/gitlab_ci_after_script.sh test_memcache: stage: testall @@ -259,53 +135,10 @@ test_memcache: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-memcached php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet memcached netcat-openbsd - - phpenmod memcached - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed 's.//memcache ..g' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/myempty-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - /etc/init.d/memcached start - - useradd testrunner - # testrunner needs to be able to read /var/log/apache2/regression-error.log for the memcache tests. - - adduser testrunner adm + - testing/gitlab_ci_script.sh memcache - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="regression-suite binding carddav scheduling" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_memcache - - xz apache2_log/test_memcache/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_memcache + - testing/gitlab_ci_after_script.sh test_ldap: stage: testldap @@ -320,48 +153,7 @@ test_ldap: when: always script: - - apt-get -y update - - bash -c 'mkdir -p /usr/share/man/man{0..10}' - - apt-get -y install locales - - echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen - - locale-gen - - echo "LANG=en_NZ.UTF-8" > /etc/default/locale - - apt-get -y install libdbd-pg-perl libyaml-perl php php-cli php-pgsql php-xml php-ldap php-curl postgresql-client postgresql libapache2-mod-php curl xmlstarlet libnet-ldap-server-test-perl netcat-openbsd - - phpenmod ldap - - curl 'https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz' | tar zxf - - - mv awl-master /usr/share/awl/ - - chown -R www-data /usr/share/awl/ - - dpkg --ignore-depends=libawl-php -i *.deb - - echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts - - rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf && touch /etc/apache2/ports.conf - - cp testing/apache-site.conf.example /etc/apache2/sites-enabled/davical-regression.conf - - sed -i 's/\/path\/to/\/usr\/share/g' /etc/apache2/sites-enabled/davical-regression.conf - - mkdir /usr/share/davical/testing/ - - cp testing/*.php /usr/share/davical/testing/ - - mkdir /usr/share/davical/htdocs/testfiles - - ln -s /usr/share/davical/htdocs/testfiles htdocs - - rm /etc/davical/config.php - - cat testing/regression-conf.php.example | sed 's.//$c->dbg.$c->dbg.' | sed -E 's.//(memcache|ldap) ..g' > /etc/davical/regression-conf.php - - ln -s /etc/davical/regression-conf.php /etc/davical/mycaldav_ldap-conf.php - - mkdir -p /var/log/davical - - chown www-data /var/log/davical - - sed -i '/peer/d' /etc/postgresql/16/main/pg_hba.conf - - echo 'local all all trust' >> /etc/postgresql/16/main/pg_hba.conf - - pg_ctlcluster 16 main start - - su postgres -c 'createuser davical_dba --createdb --createrole --superuser' - - su postgres -c 'createuser davical_app --superuser' - - su postgres -c 'createuser testrunner --superuser' - - pg_ctlcluster 16 main restart - - a2enmod rewrite - - a2enmod headers - - apache2ctl start - - useradd testrunner - # testrunner needs to be able to read /var/log/apache2/regression-error.log for the ldap tests. - - adduser testrunner adm + - testing/gitlab_ci_script.sh ldap - cd testing && su testrunner -c 'IS_CI=yes ALLSUITES="ldap" ./run_regressions.sh all x' after_script: - - mkdir -p apache2_log - - cp -r /var/log/apache2 apache2_log/test_ldap - - xz apache2_log/test_memcache_and_ldap/* - - mkdir -p davical_log - - cp -r /var/log/davical davical_log/test_ldap + - testing/gitlab_ci_after_script.sh diff --git a/testing/gitlab_ci_after_script.sh b/testing/gitlab_ci_after_script.sh new file mode 100755 index 00000000..b581432c --- /dev/null +++ b/testing/gitlab_ci_after_script.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Order: commandline, $CI_JOB_NAME, unknown +test="${1:-$CI_JOB_NAME}" +test="${test:-unknown}" + +if [ -d /var/log/apache2 ]; then + mkdir -p apache2_log + cp -r /var/log/apache2 "apache2_log/$test" + xz apache2_log/"$test"/* +fi + +if [ -d /var/log/davical ]; then + mkdir -p davical_log + cp -r /var/log/davical "davical_log/$test" +fi diff --git a/testing/gitlab_ci_script.sh b/testing/gitlab_ci_script.sh new file mode 100755 index 00000000..4a7fbbb7 --- /dev/null +++ b/testing/gitlab_ci_script.sh @@ -0,0 +1,239 @@ +#!/bin/bash + +# Configure the virtual environment that Gitlab provides for our CI jobs. + +set -e +set -x + +mode=${1:-$CI_JOB_NAME} + +ldap=0 +memcache=0 +interop=0 + +if [[ $mode =~ "ldap" ]]; then + ldap=1 + memcache=1 +elif [[ $mode =~ "memcache" ]]; then + memcache=1 +elif [[ $mode =~ "interop" ]]; then + interop=1 +fi + +# The images don't have man page directories? +mkdir -p /usr/share/man/man{0..10} + +# When we install the davical deb, ignore libawl-php version dependency, as +# we also install the libawl-php .deb built by the AWL CI. +dpkg_ignore_depends="libawl-php" + +# Images have no package info, load it. +apt-get -y update + +### +### Pre-configure the locale so it is ready before other packages are +### installed. +### + +apt-get -y install locales +echo "en_NZ.UTF-8 UTF-8" >> /etc/locale.gen +locale-gen +echo "LANG=en_NZ.UTF-8" > /etc/default/locale + +### +### Install packages required for testing +### + +packages="libdbd-pg-perl libyaml-perl postgresql-client postgresql + curl xmlstarlet netcat-openbsd" + +if [[ $mode =~ "latestphp" ]]; then + # PHP pgsql package is built from source, needs libpq-dev. + packages="$packages libpq-dev" +else + # The latestphp images already have some of these available, and others need + # to installed in another method. + packages="$packages libapache2-mod-php php php-cli php-pgsql php-xml + php-curl" +fi + +if [ $memcache == 1 ]; then + packages="$packages php-memcached memcached" +fi + +if [ $ldap == 1 ]; then + packages="$packages php-ldap libnet-ldap-server-test-perl" +fi + +if [ $interop == 1 ]; then + packages="$packages composer phpunit" +fi + +apt-get -y install $packages + +### +### Setup PHP for we're using latestphp... +### + +if [[ $mode =~ "latestphp" ]]; then + # The image doesn't install debs, so ignore the dependencies. + dpkg_ignore_depends="$dpkg_ignore_depends,php,php-pgsql,php-xml,php-cli" + + docker-php-ext-install -j$(nproc) pgsql + docker-php-ext-install -j$(nproc) pdo_pgsql + docker-php-ext-install -j$(nproc) calendar + + # The PHP 8.2 container has deflate enabled in Apache2, disable it. + # This is causes a test to fail because we're being sent a gzip + # compressed result. My reading of the curl manpage says that it should + # be decompressed by curl, but it isn't. Let's just disable the deflate + # module. + a2dismod -f deflate +fi + +### +### Setup memcache if we're using it. +### + +if [ $memcache == 1 ]; then + phpenmod memcached + /etc/init.d/memcached start + + conf_filter_memcache="| sed 's.//memcache ..g'" +fi + +### +### Prepare PHP for LDAP if we're using it. +### + +if [ $ldap == 1 ]; then + phpenmod ldap + + conf_filter_ldap="| sed 's.//ldap ..g'" +fi + +### +### Install the latest build of AWL +### + +curl https://gitlab.com/davical-project/awl/-/archive/master/awl-master.tar.gz \ + | tar zxf - +mv awl-master /usr/share/awl/ +chown -R www-data /usr/share/awl/ + +### +### Install the davical and libawl-php packages to test +### + +dpkg --ignore-depends=$dpkg_ignore_depends -i *.deb + +mkdir /usr/share/davical/testing/ +cp testing/*.php /usr/share/davical/testing/ +mkdir -p /var/log/davical +chown www-data /var/log/davical + +### +### Setup DAViCal config files +### + +rm /etc/davical/config.php +eval cat testing/regression-conf.php.example \ + \| sed 's.//\$c-\>dbg.\$c-\>dbg.' \ + $conf_filter_memcache \ + $conf_filter_ldap \ + > /etc/davical/regression-conf.php + +for site in mycaldav mycaldav_ldap myempty; do + ln -s /etc/davical/regression-conf.php /etc/davical/$site-conf.php +done + +### +### Setup PostgreSQL +### + +pg_ver=$(ls /etc/postgresql) +sed -i '/peer/d' /etc/postgresql/$pg_ver/main/pg_hba.conf +echo 'local all all trust' >> /etc/postgresql/$pg_ver/main/pg_hba.conf +pg_ctlcluster $pg_ver main start +su postgres -c 'createuser davical_dba --createdb --createrole --superuser' +su postgres -c 'createuser davical_app --superuser' +su postgres -c 'createuser testrunner --superuser' +# Why? You don't need to restart PG after creating users. +#pg_ctlcluster $pg_ver main restart + +### +### Setup testrunner user +### + +useradd testrunner +# testrunner needs to be able to read /var/log/apache2/regression-error.log +# for the memcache tests. +adduser testrunner adm + +### +### Configure Apache2 +### + +echo '127.0.1.1 regression mycaldav mycaldav_ldap myempty' >> /etc/hosts + +rm /etc/apache2/ports.conf /etc/apache2/sites-enabled/000-default.conf \ + && touch /etc/apache2/ports.conf + +cp testing/apache-site.conf.example \ + /etc/apache2/sites-enabled/davical-regression.conf +sed -i 's/\/path\/to/\/usr\/share/g' \ + /etc/apache2/sites-enabled/davical-regression.conf + +a2enmod rewrite +a2enmod headers +apache2ctl start + +# Ensure that the jobs that need to drop in static files for Apache to serve +# up can do so. +mkdir /usr/share/davical/htdocs/testfiles +chown testrunner /usr/share/davical/htdocs/testfiles +ln -s /usr/share/davical/htdocs/testfiles htdocs + +### +### Set up for carddavclientinterop +### + +if [ $interop == 1 ]; then + curl https://codeload.github.com/mstilkerich/carddavclient/tar.gz/master | tar zxf - + + cd carddavclient-master + composer install + + cat < tests/Interop/AccountData.php + [ + "username" => "user3", + "password" => "user3", + "discoveryUri" => "http://regression", + "featureSet" => TestInfrastructureSrv::SRVFEATS_DAVICAL, + "syncAllowExtraChanges" => false, + ], + ]; + + public const ADDRESSBOOKS = [ + "Davical_0" => [ + "account" => "Davical", + "url" => "http://regression/caldav.php/user3/addresses/", + "displayname" => "user3 addresses", + "description" => null, + ], + ]; +} +EOF + + mkdir -p testreports/interop +fi