New docker worflow

.github/workflows/build_dockers.yml

@@ -0,0 +1,157 @@
+name: Build Navitia Dockers
+
+on:
+  push:
+    branches:
+      - dev
+    tags:
+      - '*'
+env:
+  debian_version: debian8
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    container:
+      image: navitia/${{env.debian_version}}_dev
+      volumes:
+         # Mount so we can delete files from docker and free up space (>20GB)
+          - /usr/share/dotnet:/usr/share/dotnet
+          - /usr/local/lib/android:/usr/local/lib/android
+
+
+    steps:
+    - name: Free up space
+      run: |
+        rm -rf /usr/share/dotnet/*
+        rm -rf /usr/local/lib/android/*
+        echo "Free space:"
+        df -h
+
+    - name: Install dependencies
+      run: |
+        apt update && apt install -y --force-yes zip httpie dh-python
+
+    - name: Checkout core_team_ci_tools
+      uses: actions/checkout@v3
+      with:
+        path: core_team_ci_tools
+        repository : 'hove-io/core_team_ci_tools'
+
+    - name: Setup core_team_ci_tools python environment
+      run: |
+        pip install -r core_team_ci_tools/github_artifacts/requirements.txt --user
+
+    - name: Download mimirsbrunn package
+      run: |
+        mimirsbrunn_package="debian-package-release.zip"
+        python core_team_ci_tools/github_artifacts/github_artifacts.py \
+            -o hove-io \
+            -r mimirsbrunn \
+            -t ${{secrets.access_token_github}} \
+            -w release.yml \
+            -a $mimirsbrunn_package \
+            --output-dir .
+        unzip -qo $mimirsbrunn_package
+        rm -f $mimirsbrunn_package
+
+    - name: Download cosmogony2cities package
+      run: |
+        cosmogony2cities_package="package-${{env.debian_version}}.zip"
+        python core_team_ci_tools/github_artifacts/github_artifacts.py \
+            -o hove-io \
+            -r cosmogony2cities \
+            -t  ${{secrets.access_token_github}} \
+            -w build_package.yml \
+            -a $cosmogony2cities_package \
+            --output-dir .
+        unzip -qo $cosmogony2cities_package
+        rm -f $cosmogony2cities_package
+
+    - name: Download libkeepalive
+      run: |
+        wget http://prdownloads.sourceforge.net/libkeepalive/libkeepalive-0.3.tar.gz
+
+    - name: Restore ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: ${{env.debian_version}-package
+        max-size: 2000M
+        save: ${{ github.event_name == 'push' }}
+
+    - name: Checkout navitia
+      uses: actions/checkout@v3
+      with:
+        submodules : recursive
+        path: navitia
+        fetch-depth: 0
+
+    - name: Build navitia packages
+      working-directory: navitia
+      # Will build navitia-*.deb packages in folder ../
+      run: |
+        DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -b
+
+
+    # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable
+    - name: Choose dev navitia tag
+      if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+      run: |
+        echo "navitia_tag=dev" >> $GITHUB_ENV
+
+    - name: Choose release navitia tag
+      if: startsWith(github.ref, 'refs/tags/')
+      run: |
+        cd navitia
+        version=$(git describe)
+        echo "building version $version"
+        echo "navitia_tag=$version" >> $GITHUB_ENV
+
+
+    - name: Create dockers images and push them
+      run: |
+        docker build -f navitia/docker/${{env.debian_version}}/Dockerfile-master -t navitia/master .
+
+        components='jormungandr kraken tyr-beat tyr-worker tyr-web instances-configurator mock-kraken eitri'
+        for component in $components; do
+            echo "*********  Building $component ***************"
+            docker build -t navitia/$component:${{env.navitia_tag}} -f  navitia/docker/${{env.debian_version}}/Dockerfile-${component} .
+
+            # add latest tag if navitia_tag != dev
+            if [[ "${{env.navitia_tag}}" != "dev" ]]; then
+                docker tag navitia/$component:${{env.navitia_tag}}  navitia/$component:latest
+            fi
+        done
+
+        docker login -u ${{secrets.docker_user}} -p ${{secrets.docker_password}}
+
+        for component in $components; do
+            echo "*********  Pushing $component ***************"
+            docker push --all-tags navitia/$component:${{env.navitia_tag}}
+        done
+
+
+    - name: Publish Aws images
+      uses: hove-io/navitia/.github/workflows/publish_hove_images_aws.yml@dev
+      with:
+        navitia_tag: ${{env.navitia_tag}}
+
+    - name: Run artemis on push to dev
+      if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+      uses: peter-evans/repository-dispatch@v2
+      with:
+        token: ${{ secrets.access_token_github }}
+        repository: hove-io/artemis
+        event-type: run_artemis_ng
+
+    - name: failure notification
+      if: failure()
+      run: |
+          echo '{"text":":warning: Github Actions: build_dockers for ${{env.navitia_tag}} failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_CORE_TEAM_URL}}
+
+    - name: success notification on navitia core team
+      if: success()
+      run: |
+          echo '{"text":":octopus: Github Actions: build_dockers succeeded. New navitia ${{env.navitia_tag}} image available.' | http --json POST ${{secrets.SLACK_NAVITIA_CORE_TEAM_URL}}

.github/workflows/publish_hove_images_aws.yml

@@ -16,6 +16,14 @@ on:
         default: 'dev'
         required: true
 
+  workflow_call:
+    inputs:
+      navitia_tag:
+        description: 'Tag of the Navitia image (dev or v1.2.3)'
+        type: string
+        required: true
+
+
   workflow_run:
     workflows: ["Build Navitia Packages For Release", "Build Navitia Packages For Dev Multi Distributions"]
     branches: [release, dev]
@@ -48,6 +56,15 @@ jobs:
             echo "Workflow triggered manually"
             echo "::set-output name=branch::${{ github.event.inputs.environment }}"
             echo "::set-output name=tag::${{ github.event.inputs.tag }}"
+          elif [[ '${{ github.event_name }}' == 'workflow_call' ]]
+          then
+            echo "Workflow call"
+            if [[ "${{inputs.navitia_tag}}" != "dev" ]]; then
+              echo "::set-output name=branch::release"
+            else :
+              echo "::set-output name=branch::dev"
+            fi
+            echo "::set-output name=tag::${{ inputs.navitia_tag }}"
           else :
             exit 1
           fi

docker/apache/apache2.conf

@@ -0,0 +1,50 @@
+######  # Ansible managed ######
+# Security
+TraceEnable On
+
+ServerName %{APACHE_SERVER_NAME}
+
+ServerRoot "/etc/apache2"
+PidFile ${APACHE_PID_FILE}
+Timeout 120
+KeepAlive Off
+MaxKeepAliveRequests 100
+KeepAliveTimeout 15
+LimitRequestFieldSize 8190
+
+
+User www-data
+Group www-data
+
+AccessFileName .htaccess
+<FilesMatch "^\.ht">
+    Require all denied
+</FilesMatch>
+
+<Directory />
+  Options FollowSymLinks
+  AllowOverride None
+</Directory>
+
+HostnameLookups Off
+ErrorLog /var/log/apache2/error.log
+LogLevel warn
+EnableSendfile On
+
+#Listen 80
+
+
+Include "/etc/apache2/mods-enabled/*.load"
+Include "/etc/apache2/mods-enabled/*.conf"
+Include "/etc/apache2/ports.conf"
+
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %b" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"" forwarded
+LogFormat "{\"time\":\"%{%s}t.%{usec_frac}t\", \"bytes_in\":\"%I\", \"bytes_out\":\"%O\", \"cookie\":\"%{Cookie}i\", \"server\":\"%v\", \"dest_port\":\"%p\", \"http_content_type\":\"%{Content-type}i\", \"http_method\":\"%m\", \"http_referrer\":\"%{Referer}i\", \"http_user_agent\":\"%{User-agent}i\", \"ident\":\"%l\", \"response_time_microseconds\":\"%D\", \"client\":\"%{X-Forwarded-For}i\", \"status\":\"%>s\", \"uri_path\":\"%U\", \"uri_query\":\"%q\", \"user\":\"%u\"}" splunk_json
+
+IncludeOptional conf-enabled/*.conf
+IncludeOptional "/etc/apache2/sites-enabled/*"
+

docker/apache/envvars

@@ -0,0 +1,47 @@
+# envvars - default environment variables for apache2ctl
+
+# this won't be correct after changing uid
+unset HOME
+
+# for supporting multiple apache2 instances
+if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then
+	SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}"
+else
+	SUFFIX=
+fi
+
+# Since there is no sane way to get the parsed apache2 config in scripts, some
+# settings are defined via environment variables and then used in apache2ctl,
+# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc.
+export APACHE_RUN_USER=www-data
+export APACHE_RUN_GROUP=www-data
+# temporary state file location. This might be changed to /run in Wheezy+1
+export APACHE_PID_FILE=/var/run/apache2/apache2$SUFFIX.pid
+export APACHE_RUN_DIR=/var/run/apache2$SUFFIX
+export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX
+# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2.
+export APACHE_LOG_DIR=/var/log/apache2$SUFFIX
+
+## The locale used by some modules like mod_dav
+export LANG=C
+## Uncomment the following line to use the system default locale instead:
+#. /etc/default/locale
+
+export LANG
+
+## The command to get the status for 'apache2ctl status'.
+## Some packages providing 'www-browser' need '--dump' instead of '-dump'.
+#export APACHE_LYNX='www-browser -dump'
+
+## If you need a higher file descriptor limit, uncomment and adjust the
+## following line (default is 8192):
+#APACHE_ULIMIT_MAX_FILES='ulimit -n 65536'
+
+## If you would like to pass arguments to the web server, add them below
+## to the APACHE_ARGUMENTS environment.
+#export APACHE_ARGUMENTS=''
+
+## Enable the debug mode for maintainer scripts.
+## This will produce a verbose output on package installations of web server modules and web application
+## installations which interact with Apache
+# export APACHE2_MAINTSCRIPT_DEBUG=1

docker/apache/jormungandr.default.conf

@@ -0,0 +1,37 @@
+######  # Ansible managed ######
+
+<VirtualHost *:80>
+    ServerName %{APACHE_SERVER_NAME}
+    ServerAlias %{APACHE_SERVER_ALIAS}
+
+    Timeout 30
+    SetEnv PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION cpp
+    <Location /usr/src/app>
+        Require all granted
+    </Location>
+
+    AllowEncodedSlashes On
+
+    RewriteEngine on
+    RewriteCond %{HTTP:Authorization} ^(.*)
+    RewriteRule .* - [e=HTTP_AUTHORIZATION:%1]
+
+    RewriteCond %{REQUEST_URI} ^(.*)//(.*)$
+    RewriteRule . %1/%2  [L,PT]
+
+    AddOutputFilterByType DEFLATE application/json
+
+    CustomLog /var/log/apache2/access.log vhost_combined_timetaken
+
+    KeepAlive Off
+
+    ProxyRequests Off
+    ProxyPreserveHost On
+    SetEnv proxy-nokeepalive 1
+    SetEnv force-proxy-request-1.0 1
+    ProxyPass /server-status !
+    ProxyPass / http://127.0.0.1:9090/ retry=0 timeout=30
+    ProxyPassReverse / http://127.0.0.1:9090/
+
+</VirtualHost>
+

docker/apache/logformat.conf

@@ -0,0 +1,17 @@
+######  # Ansible managed ######
+# Add:
+# - timetaken %D in microseconds
+# - 'Host' header: %{Host}i
+# http://httpd.apache.org/docs/2.2/en/mod/mod_log_config.html#formats
+LogFormat "%h %l %u %t [APACHE] \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D %{Host}i" vhost_combined_timetaken
+# Same as above, but log the PHPSESSID cookie value to identify different users
+# is they come from same ip (proxy)
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D %{Host}i %{PHPSESSID}C" vhost_combined_timetaken_phpsessionid
+
+# Same as above, but log the JSESSIONID cookie value to identify different users
+# is they come from same ip (proxy)
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D %{Host}i %{JSESSIONID}C" vhost_combined_timetaken_jsessionid
+# Same as above, but log the cookie value
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D %{Host}i \"%{Cookie}i\"" vhost_combined_timetaken_cookie
+
+

docker/ca-certificates/pca-hove.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFUzCCAzugAwIBAgIRAPPkzEJyFXz2JGWaGHmMJ7IwDQYJKoZIhvcNAQENBQAw
+QzELMAkGA1UEBhMCRlIxEDAOBgNVBAoMB0hvdmUuaW8xDDAKBgNVBAsMA0RTSTEU
+MBIGA1UEAwwLYXdzLnByaXZhdGUwHhcNMjIwNTI0MTE0MDQ2WhcNMzcwNTI0MTI0
+MDQ2WjBDMQswCQYDVQQGEwJGUjEQMA4GA1UECgwHSG92ZS5pbzEMMAoGA1UECwwD
+RFNJMRQwEgYDVQQDDAthd3MucHJpdmF0ZTCCAiIwDQYJKoZIhvcNAQEBBQADggIP
+ADCCAgoCggIBAMbj58sKK8wympczYkkTZof5PNaAvJH8mde17dV2bn7fLFtP/Wi4
+aD7qwle1r8X02rHQIxkTfNr6UnmEuTyLRtQh7nYPyChtns2lE+NcI30Mg1NXfSm8
+UPdpscsOKLkX3b5RcpbbhbeqdGlnOKq13rn4gdr3oIoNxilxDwmRffbEhWdSAhHQ
+FuiosSFlPZdq64xFlSp2R5ThhDA7a26oZfuiZYiJY33iDU22cMkkP9K+3CufPZIH
+3iPYPzNciKlOPBDvpfRA/3vCyR/VhegEjolozOO7xd8uW6VYJWm63bZtoYplJa4Z
+KVx2PAuMESPHPzaPFVXh2/i9e2mfEf4wFJX9jU7zchW1d2AvdUtcxgkTJnWC1Qha
+7DkBHWHUpuw6+4LFsFVYyLW0xVY3t+NtD2jXA5co7lBEvGCSCHgGPuJAv6eUBWUe
+7bGpEJq8alr/7AnOQWaR3lf10D4VhL/aYAAY881EKz4/9x5EFRZnNNhmw8R2xD9r
+ZXHwCotST5iSWcKsYHgTMdssIxvxIvHznnPxj7YPpuUWbi9f3FOlldvxE/+NIqNi
+OduX/PJ/X+YlkAgjkQDHeUZlnAnGYFTnnTnxS9sq6eeLTKmnj9fYQegglMHfD4I+
+Ahcg652uwX5OJEWw/Eu7zRea1z9h4fLCT0ytXE2IKLaHO20q5YtADiJnAgMBAAGj
+QjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNIic6flCwfj9X3VMgoP3Vly
+fu2oMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQ0FAAOCAgEADEMXmvE5I8ZD
+TUMif1S7cwVhDhWNRavyFTTSZuQ370gF8AlhJxMDVO8u+PCTiLgYPdXRvQYiY0de
+pe0rVzJTLLzp0C2N6HndnhbDFyBw4hFnrvoJlYfEMyneiqIUsGWemLr7eU9MFRGL
+/1UQu48FxR3nzRFgETn2tJW9/DOQHTvZqXCnZte/Q3nS4Iz8s7IAehA/XnvGTFxX
+qLQwOKOSf8/5uVxyjtgyr1Z0KZ3p0W/HMihDKvcflFGITxGMJyHGhRj9D1c3QQKr
+rplsKVfRr61z2GbyIhpktNsUYY14O6XK/wgcQfniePV7I/sjy3lFaqijW7//4QB4
+sLyS8wCsjuLfQFALHEzVaOlepb4BdBcF8XIuM1zpbtbCVe8eG809bCABlRwZf7Bp
+z84qVrTfs+Lf6w/PBqX2tHNXnM1mHjBeymMAWS91Uqpo8I21PvH4kLrGzrzx2d+Q
+Nz+LMUquAeLsh8B/GT2jgdobbEB1tOrF7ojRRKKJl+Aa9BEgxRU5/8DAtGixOWIN
+OjjLNsx8fb/yFtsGYleDmLnQSqIztGdcDDCAjxop0W6+Cf1kdyhzthsfmizlD5N9
+r76uVwKp6FYQxAThEbWzphWxGPoOgVQGDX4PqhqIAy/CjX4/MLaJvvmQUN1An1te
+WRJz05Q3y4UkRP6LTBtG3LFsmei69gg=
+-----END CERTIFICATE-----

docker/debian10/Dockerfile-eitri

@@ -0,0 +1,15 @@
+FROM navitia/master
+
+# copy package from context inside the docker
+COPY navitia-ed_*.deb /
+COPY navitia/source/ /navitia/source/
+
+# install dependencies
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip libpq-dev jq git zip curl \
+    && apt-get install -y /navitia-ed_*.deb \
+    && apt-get clean \
+    && rm -rf /navitia-ed_*.deb
+
+# install eitri requirements
+RUN pip3 install --no-cache-dir -r /navitia/source/eitri/requirements.txt