# Automatisierte Builds mit GitLab CI/CD
Wir nutzen einen selbstgehostete GitLab Runner (opens new window) auf einer EC2-Instance (t2.micro) (opens new window) (staging-Server)
Gitlab-Runner-Manager - EC2 (
1 GB RAM, 1 vCPU, 20 GB SSD)
Der Runner mit dem Token _mbpN6UZ... nutzt den Docker+Machine-Executer (opens new window).
Das bedeutet, er spawned für jedes Build/Job neue EC2-Instanzen (c5.large) und beendet diese nach dem Build wieder. Momentan sind es max 5 parallel.
Der Runner baut das NewSite, Wiki und Styleguide-Projekt (zukünftigalle anderen Projekte auch).
# Buildprozess
Der Buildprozess besteht aus den Stages:
.pre- composer
- npm
build- build-assets
- db-seeding
test- dependency scanning
- dusk
- phpcpd
- phpunit
- sensiolabs
deploy- deploy Staging
.post
Er hat einen sogenannten Directed Acyclic Graph (opens new window).
# Graphen des Buildprozesses
DAG Graph

Pipeline Visualisiert

⚠️ es kann immer eine neuere Version der .gitlab-ci.yml geben.
Bitte hier gucken: https://gitlab.com/placing-you/NewSite/-/blob/main/.gitlab-ci.yml
# .gitlab-ci.yml
# https://github.com/ohdearapp/gitlab-ci-pipeline-for-laravel/blob/master/.gitlab-ci.yml
image: chilio/laravel-dusk-ci:php-8.1
include:
template: Verify/Browser-Performance.gitlab-ci.yml
# Variables (will be overwirtten by defined Variables in GitLab (https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables))
variables:
MYSQL_ROOT_PASSWORD: root
MYSQL_USER: mysql_user
MYSQL_PASSWORD: mysql_password
MYSQL_DATABASE: mysql_db
MYSQL_HOST: mysql
REDIS_PORT: 6379
# cache:
# key: '$CI_JOB_NAME-$CI_COMMIT_REF_SLUG' # Caching Strategy taken from https://github.com/ohdearapp/gitlab-ci-pipeline-for-laravel/blob/master/.gitlab-ci.yml
# Speed up builds
cache:
key: $CI_COMMIT_REF_NAME # changed to $CI_COMMIT_REF_NAME in Gitlab 9.x
# Add a `.` in front of a job to make it hidden.
# Add a `&reference` to make it a reusable template.
# Note that we don't have dashes anymore.
.init_ssh_staging: &init_ssh_staging |
mkdir -p ~/.ssh
echo -e "$PM_SSH_PRIVATE_KEY_staging" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
# Add a `.` in front of a job to make it hidden.
# Add a `&reference` to make it a reusable template.
# Note that we don't have dashes anymore.
.init_ssh_production: &init_ssh_production |
mkdir -p ~/.ssh
echo -e "$PM_SSH_PRIVATE_KEY_production" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
composer:
image: thecodingmachine/php:8.1-v4-slim-cli
stage: .pre
needs: []
only:
- merge_requests
- tags
- main
variables:
IS_BUILDING: 'true'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- php -v
- bash scaffoldLaravelStorage.sh
- bash createEnvGitlabRunner.sh
- composer --version
- composer config http-basic.nova.laravel.com ${env_NOVA_USERNAME} ${env_NOVA_LICENSE_KEY}
- sudo composer install --prefer-dist --no-ansi --no-interaction --no-progress --ignore-platform-reqs
artifacts:
paths:
- vendor
- storage
- public/vendor
- public/svg
- bootstrap/cache
expire_in: 7 days
when: always
cache:
paths:
- vendor
environment:
name: build
npm:
image: node:16
stage: .pre
needs: []
only:
- merge_requests
- tags
- main
variables:
IS_BUILDING: 'true'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- npm --version
- npm install
- npm run writeTailwindJson
artifacts:
paths:
- node_modules
- public/tailwind.config.json
expire_in: 7 days
when: always
cache:
paths:
- node_modules
environment:
name: build
build-assets staging:
stage: build
# Download the artifacts for these jobs
needs: ['composer', 'npm']
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
env_MYSQL_ROOT_PASSWORD: root
env_MYSQL_USER: mysql_user
env_MYSQL_PASSWORD: mysql_password
env_MYSQL_DATABASE: mysql_db
env_MYSQL_HOST: mysql
IS_BUILDING: 'true'
before_script:
- apt update && apt install -y libjpeg62
script:
- php -v
- bash createEnvGitlabRunner.sh
- chmod -R 777 public
- npm --version
- node --version
- php artisan about
- php artisan airdrop:download
- npm run prod
- php artisan airdrop:upload
artifacts:
paths:
- public
- .env
expire_in: 1 days
when: always
environment:
name: staging
build-assets production:
stage: build
# Download the artifacts for these jobs
needs: ['composer', 'npm']
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
env_MYSQL_ROOT_PASSWORD: root
env_MYSQL_USER: mysql_user
env_MYSQL_PASSWORD: mysql_password
env_MYSQL_DATABASE: mysql_db
env_MYSQL_HOST: mysql
IS_BUILDING: 'true'
before_script:
- apt update && apt install -y libjpeg62
script:
- php -v
- bash createEnvGitlabRunner.sh
- chmod -R 777 public
- npm --version
- node --version
- php artisan about
- php artisan airdrop:download
- npm run prod
- php artisan airdrop:upload
artifacts:
paths:
- public
- .env
expire_in: 1 days
when: always
environment:
name: production
db-seeding:
stage: build
needs: ['composer']
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
services:
- name: mysql:8.0
command: ['--default-authentication-plugin=mysql_native_password', '--sort_buffer_size=128MB']
- name: redis:6.0
command: ['redis-server']
variables:
QUEUE_CONNECTION: sync
IS_BUILDING: 'true'
script:
# fixes the problem that PDF cant be converted via imagick
- sed -i 's~<policy domain="coder" rights="none" pattern="PDF" />~<!---<policy domain="coder" rights="none" pattern="PDF" />-->~g' /etc/ImageMagick-6/policy.xml
- php -v
- mkdir -p public/media
- bash createEnvGitlabRunner.sh
- php artisan about
- php artisan key:generate
- php artisan db:test mysql
- php artisan db:version
- php artisan migrate:fresh
- php artisan db:seed
- mysqldump --host="${MYSQL_HOST}" --user="${MYSQL_USER}" --password="${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" --no-tablespaces --routines --triggers > db.sql
artifacts:
paths:
- storage/logs # for debugging
- storage/app/public # for the generated images
- public/media # the images of the media-library
- db.sql
#larastan:
# stage: test
# script:
# - ./vendor/bin/phpstan analyse --memory-limit=2G
insights:
stage: test
needs: ['composer']
only:
- merge_requests
- tags
- main
variables:
IS_BUILDING: 'true'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- vendor/bin/phpinsights --verbose
- vendor/bin/phpinsights -n --ansi --format=codeclimate > codeclimate-report.json
artifacts:
reports:
codequality: codeclimate-report.json
lint:
image: node
stage: test
needs: ['npm']
only:
- merge_requests
- tags
- main
variables:
IS_BUILDING: 'true'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- npm run lint
phpcpd:
stage: test
needs: []
only:
- merge_requests
- tags
- main
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
IS_BUILDING: 'true'
script:
- composer phpcpd
cache:
paths:
- phpcpd.phar
sensiolabs:
image: cirrusci/wget
stage: test
needs: []
only:
- merge_requests
- tags
- main
variables:
IS_BUILDING: 'true'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- wget -O local-php-security-checker $SECURITY_CHECKER_URL
- chmod +x local-php-security-checker
- ./local-php-security-checker
cache:
paths:
- security-checker/
dependency scanning:
stage: test
needs: []
only:
- merge_requests
- tags
variables:
IS_BUILDING: 'true'
APP_URL: 'http://localhost'
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
script:
- npm audit --prod
pest:
stage: test
needs: ['composer', 'build-assets staging']
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
services:
- name: mysql:8.0
command: ['--default-authentication-plugin=mysql_native_password']
# Download the artifacts for these jobs
script:
- bash createEnvGitlabRunner.sh
# fixes the problem that PDF cant be converted via imagick
- sed -i 's~<policy domain="coder" rights="none" pattern="PDF" />~<!---<policy domain="coder" rights="none" pattern="PDF" />-->~g' /etc/ImageMagick-6/policy.xml
- php -v
- php artisan about
- php artisan key:generate
- php artisan db:test
- php artisan db:version
- php artisan migrate
- ./vendor/bin/pest --version
- php artisan test
artifacts:
paths:
- storage/
- tests/Browser/screenshots
- tests/Browser/console
- vendor/
- public/
- bootstrap/cache
expire_in: 7 days
when: always
retry: 2
environment:
name: testing
dusk:
stage: test
needs: ['composer', 'db-seeding', 'build-assets staging']
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
PHP_MEMORY_LIMIT: 128M
services:
- name: redis:6.0
command: ['redis-server']
- name: mysql:8.0
command: ['--default-authentication-plugin=mysql_native_password', '--sort_buffer_size=128MB']
script:
- bash createEnvGitlabRunner.sh
- bash scaffoldLaravelStorage.sh
- php -v
# fixes the problem that PDF cant be converted via imagick
- sed -i 's~<policy domain="coder" rights="none" pattern="PDF" />~<!---<policy domain="coder" rights="none" pattern="PDF" />-->~g' /etc/ImageMagick-6/policy.xml
- php artisan about
- php artisan key:generate
- php artisan db:test
- php artisan db:version
- mysql --host="${MYSQL_HOST}" --user="${MYSQL_USER}" --password="${MYSQL_PASSWORD}" -e "SELECT @@version;"
- mysql --host="${MYSQL_HOST}" --user="${MYSQL_USER}" --password="${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" < db.sql
- chrome-system-check
- start-nginx-ci-project
- composer dusk
artifacts:
paths:
- storage/
- tests/Browser/screenshots
- tests/Browser/console
- vendor/
- public/
- bootstrap/cache
expire_in: 7 days
when: always
environment:
name: testing
deployment Staging:
stage: deploy
image: extrameile/php-deployment
# Download the artifacts for these jobs
needs: ['build-assets staging', 'composer', 'db-seeding', 'lint', 'phpcpd', 'sensiolabs', 'dependency scanning']
only:
- tags
- merge_requests
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
IS_BUILDING: 'true'
before_script:
# reference the hidden function from the top
- *init_ssh_staging
script:
- bash createEnvGitlabRunner.sh
- mkdir -p public/media
- echo "CI_JOB_ID=$CI_JOB_ID" >> .env
- ssh forge@$SERVER_ADDRESS "chmod -R 777 /home/forge/${env_APP_DOMAIN}"
- ssh forge@$SERVER_ADDRESS "php /home/forge/${env_APP_DOMAIN}/artisan down --retry=60 || true"
# stop horizon
- ssh forge@$SERVER_ADDRESS "php /home/forge/${env_APP_DOMAIN}/artisan horizon:terminate"
- rm -rf resources/{css,fonts,js,styleguide}
- ssh forge@$SERVER_ADDRESS "rm -rf /home/forge/${env_APP_DOMAIN}/public/media"
- rsync -avPrq --delete .env app artisan bootstrap config db.sql public composer.json composer.lock database lang resources routes server.php scaffoldLaravelStorage.sh package.json storage vendor forge@$SERVER_ADDRESS:/home/forge/${env_APP_DOMAIN}
- rsync -avPrq --delete .env forge@$SERVER_ADDRESS:/home/forge
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && bash /home/forge/${APP_DOMAIN}/scaffoldLaravelStorage.sh'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan about'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan db:test'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan key:generate'
# testing getting the mysql Version
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && cd /home/forge/${APP_DOMAIN} && mysql --host="${DB_HOST}" --user="${DB_USERNAME}" --password="${DB_PASSWORD}" -e "SELECT @@version;"'
# Backup current Database
# FIXME: find a better backup process
#- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan backup:run --only-db'
# import saved db.sql (seeded data)
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && cd /home/forge/${APP_DOMAIN}/ && mysql --host="${DB_HOST}" --user="${DB_USERNAME}" --password="${DB_PASSWORD}" "${DB_DATABASE}" < db.sql'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan sitemap:generate'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan cache:clear'
# cache certain things to make the app faster
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && cd /home/forge/${APP_DOMAIN} && composer dump-autoload -o || true'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan event:cache'
- ssh forge@$SERVER_ADDRESS "sudo service php8.1-fpm reload"
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan about'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan media:move || true'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan config:cache'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan event:cache'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan optimize'
# exit maintenance mode
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan up || true'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan data:sync-feedback'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan health:check'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan health:list'
- ssh forge@$SERVER_ADDRESS 'rm -rf /home/forge/.env'
environment:
name: staging
url: $env_APP_URL
artifacts:
paths:
- .env
expire_in: 7 days
when: always
when: manual
deployment Production:
stage: deploy
# Download the artifacts for these jobs
needs: [
'build-assets production',
'composer',
#'db-seeding',
'lint',
'phpcpd',
'sensiolabs',
'dependency scanning',
# 'pest',
# 'dusk',
]
only:
- tags
- merge_requests
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
IS_BUILDING: 'true'
before_script:
# reference the hidden function from the top
- *init_ssh_production
script:
- rm -rf .env
- bash createEnvGitlabRunner.sh
- chmod 777 .env
- php artisan key:generate
- echo "CI_JOB_ID=$CI_JOB_ID" >> .env
- ssh forge@$SERVER_ADDRESS "php /home/forge/${env_APP_DOMAIN}/artisan down --retry=60 || true"
- ssh forge@$SERVER_ADDRESS "php /home/forge/${env_APP_DOMAIN}/artisan horizon:terminate"
- rm -rf resources/{css,fonts,js,styleguide}
- rsync -avPrq --delete .env app artisan database bootstrap config public composer.json composer.lock lang resources routes scaffoldLaravelStorage.sh server.php package.json vendor forge@$SERVER_ADDRESS:/home/forge/${env_APP_DOMAIN}
- rsync -avPrq --delete .env forge@$SERVER_ADDRESS:/home/forge
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && bash /home/forge/${APP_DOMAIN}/scaffoldLaravelStorage.sh'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan about'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan db:test'
# Backup current Database
# FIXME: find a better backup process
# - ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan backup:run --only-db'
# import saved db.sql (seeded data)
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan sitemap:generate'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan cache:clear'
# cache certain things to make the app faster
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && cd /home/forge/${APP_DOMAIN} && composer dump-autoload -o || true'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan config:cache'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan event:cache'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan optimize'
- ssh forge@$SERVER_ADDRESS 'sudo service php8.1-fpm reload'
# exit maintenance mode
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan up || true'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan data:sync-feedback'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan health:check'
- ssh forge@$SERVER_ADDRESS 'source /home/forge/.env && php /home/forge/${APP_DOMAIN}/artisan health:list'
- ssh forge@$SERVER_ADDRESS 'rm -rf /home/forge/.env'
environment:
name: production
url: $env_APP_URL
artifacts:
paths:
- .env
expire_in: 7 days
when: always
when: manual
browser_performance:
# https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html
image: docker:git
stage: .post
only:
- merge_requests
- tags
except:
variables:
- $CI_MERGE_REQUEST_TITLE =~ /^WIP:.*/
- $CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/
variables:
URL: $env_APP_URL
SITESPEED_VERSION: 14.1.0
SITESPEED_OPTIONS: ''
DOCKER_TLS_CERTDIR: ''
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://thedockerhost:2375/
services:
- name: docker:stable-dind
alias: thedockerhost
script:
- mkdir gitlab-exporter
- wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js
- mkdir sitespeed-results
- docker info
- docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
- mv sitespeed-results/data/performance.json browser-performance.json
artifacts:
reports:
performance: browser-performance.json
environment:
name: production
url: $URL
# querverweisende Links
*2.1.1 Server
# weiterführende Links
*2.5.1 Release