-
Notifications
You must be signed in to change notification settings - Fork 142
340 lines (336 loc) · 17.9 KB
/
publish_v3.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
# Publish SPDX specification to https://spdx.github.io/spdx-spec/
#
# This workflow is configured differently for the
# "main", "develop", and "support" branches, each publishing to a unique URL.
#
# For example,
# the workflow in "main" may publish to https://spdx.github.io/spdx-spec/v3.0.1/,
# the workflow in "develop" may publish to https://spdx.github.io/spdx-spec/v3.1-dev/,
# the workflow in "support/3.0" may publish to https://spdx.github.io/spdx-spec/v3.0.x/.
#
# The workflow should be configured to have an URL without a version number
# specified be redirected to an URL published from "main" branch.
#
# ## Workflow overview
#
# 1) Generate model documents and RDFs from model files in spdx-3-model repo
# 2) Combine the model documents from (1) with the chapters in spdx-spec repo
# 3) Generate a website using files from (2)
# 4) Upload RDFs from (1) and a website from (3) to GitHub Pages
# 5) Make URL redirections as needed
#
# See notes at:
# https://github.com/spdx/spdx-spec/issues/1155
# https://github.com/spdx/spdx-spec/pull/1146
# See branch structure at:
# https://github.com/spdx/spdx-spec/blob/develop/README.md#branch-structure
on:
push:
branches:
- develop # This should match with REF_SPEC,
# to automatically publish from a correct branch
repository_dispatch:
types:
- publish_v3_spec
workflow_dispatch: {} # Manually trigger from https://github.com/spdx/spdx-spec/actions
jobs:
build:
runs-on: ubuntu-latest
env:
REF_SPEC: "develop" # spdx-spec branch: "main" or "develop" or "support/x.y" or tag/commit ID
REF_MODEL: "develop" # spdx-3-model branch: "main" or "develop" or "support/x.y" or tag/commit ID
REF_PARSER: "main" # spec-parser branch: "main" or tag/commit ID
GH_PAGES_BRANCH: "gh-pages" # spdx-spec branch to publish HTML to
VERSION_DEFAULT: "v3.0.1" # Default version:
# - A version to be redirected to from the URL without
# a version number specified
# - Should be the latest stable version from the "main" branch
# - VERSION_DEFAULT should be the same across all
# branches/tags
# - VERSION_DEFAULT should also match the
# mike's canonical_version in mkdocs.yml
VERSION: "v3.1-dev" # Publishing version, to be published from this branch:
# - VERSION can be different from VERSION_DEFAULT;
# For example, if VERSION is a draft/release candidate,
# or if VERSION is a stable version that is behind the
# default version (e.g. v3.0.2 vs v3.1)
# - VERSION from "develop" branch should be indicated with
# a suffix ("-dev", "-draft", etc.).
# The content of this version will constantly change.
# - VERSION should match with the version in the copyright
# text defined in mkdocs.yml
# e.g. "SPDX v3.x.x Copyright (c) 2010-2024, ..."
# - A release candidate (with suffix "-RC") may be published
# from a very short-lived "support" branch.
# The content of this version should be kept unchanged,
# so it can be properly referenced during the review period,
# but the URL of the RC version may subjected to be
# redirected to the release version later.
# For example, v3.0-RC1 was redirected to v3.0 and
# will be redirected to v3.0.1 later.
VERSION_ALIASES: "v3.1"
# VERSION_ALIASES are names that will be redirected to VERSION
# - Can be empty, can be multiple; separated by space
# - "latest" should be reserved for the latest version
# - Versions like "v3.0" will be expanded to "v3.0 3.0"
GIT_USER_NAME: "ci-bot" # Username for gh-pages commit
GIT_USER_EMAIL: "[email protected]" # E-mail for gh-pages commit
MKDOCS_MODEL_YML: "model-files.yml" # Contains list of model Markdown files:
# - relative to PARSER_OUT_BASE_DIR
MKDOCS_BASE_YML: "mkdocs.yml" # Initial MkDocs configuration; from spdx-spec repo
MKDOCS_FULL_YML: "__mkdocs-full.yml" # MkDocs configuration combined with model list:
# - to be generated from MKDOCS_BASE_YML and MKDOCS_MODEL_YML
REDIRECT_MAP_PATH: "etc/redirect-map.csv" # URL redirect map
REDIRECT_TEMPLATE_PATH: "etc/redirect-template.html" # URL redirect HTML template
steps:
- name: Expand version aliases to include a version without 'v' prefix
# For example, given
# - VERSION: "v3.0.1"; and
# - Original VERSION_ALIASES: "latest v3.0"
# the expanded VERSION_ALIASES will be "3.0.1 latest v3.0 3.0"
run: |
echo VERSION: $VERSION
echo Original VERSION_ALIASES: $VERSION_ALIASES
original_aliases="$VERSION_ALIASES"
expanded_aliases=""
if [[ $VERSION =~ ^v[0-9] ]]; then
expanded_aliases="$expanded_aliases ${VERSION#v}"
fi
for version in $original_aliases; do
expanded_aliases="$expanded_aliases $version"
if [[ $version =~ ^v[0-9] ]]; then
expanded_aliases="$expanded_aliases ${version#v}"
fi
done
expanded_aliases=$(echo $expanded_aliases | sed 's/^ *//g')
echo "VERSION_ALIASES=$expanded_aliases" >> $GITHUB_ENV
- name: Check expanded version aliases
run: |
echo Expanded VERSION_ALIASES: $VERSION_ALIASES
- name: Checkout spdx-spec
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
with:
ref: ${{ env.REF_SPEC }}
path: spdx-spec
fetch-depth: 0 # Because we will be pushing the gh-pages branch
- name: Checkout spdx-3-model
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
with:
repository: spdx/spdx-3-model
ref: ${{ env.REF_MODEL }}
path: spdx-3-model
- name: Checkout spec-parser
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
with:
repository: spdx/spec-parser
ref: ${{ env.REF_PARSER }}
path: spec-parser
- name: Set up specific Python version
uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 #v5.4.0
with:
python-version: "3.12"
cache: "pip"
- name: Install pre-requisites for spdx-spec
run: pip install -r spdx-spec/requirements.txt
- name: Install pre-requisites for spec-parser
run: pip install -r spec-parser/requirements.txt
- name: Build model files
# Also move the model file list to spdx-spec/ directory,
# as it should not be part of the publicly accessible website.
run: |
python3 spec-parser/main.py --force --generate-rdf --output-rdf spdx-spec/docs/rdf --generate-mkdocs --output-mkdocs spdx-spec/docs/model --generate-plantuml --output-plantuml spdx-spec/docs/diagram --generate-jsondump --output-jsondump spdx-spec/docs/jsondump spdx-3-model/model
mv spdx-spec/docs/model/$MKDOCS_MODEL_YML spdx-spec/
- name: Copy JSON annotations
# JSON annotations URL will be redirected
# from https://spdx.org/rdf/<version>/spdx-json-serialize-annotations.ttl
# to https://spdx.github.io/spdx-spec/v<version>/rdf/jsonld-annotations.ttl
#
# To maintain availability of files at the old location at
# https://spdx.github.io/spdx-spec/v3.0/model/<filename>,
# another copy will be made into spdx-spec/docs/model/.
#
# Note: When release a new version, update the content of annotations.ttl to match the version
run: |
cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/rdf/jsonld-annotations.ttl
cp spdx-spec/serialization/jsonld/annotations.ttl spdx-spec/docs/model/jsonld-annotations.ttl
- name: Copy JSON-LD context and RDFs
# JSON-LD context and RDF URLs will be redirected
# from https://spdx.org/rdf/<version>/spdx-context.jsonld, spdx-model.ttl, etc.
# to https://spdx.github.io/spdx-spec/v<version>/rdf/spdx-context.jsonld, etc.
#
# Make a copy of spdx-model.json-ld as spdx-model.jsonld,
# because we have spdx-context.jsonld.
run: |
cp spdx-spec/docs/rdf/spdx-model.json-ld spdx-spec/docs/rdf/spdx-model.jsonld
cp spdx-spec/docs/rdf/spdx-context.jsonld spdx-spec/docs/model/
cp spdx-spec/docs/rdf/spdx-model.* spdx-spec/docs/model/
- name: Generate JSON schema
# JSON schema URL will be redirected
# from https://spdx.org/schema/<version>/spdx-json-schema.json
# to https://spdx.github.io/spdx-spec/v<version>/rdf/schema.json
#
# Note: When release a new version, update URL in --context-url line to match the version
run: |
shacl2code generate \
--input spdx-spec/docs/rdf/spdx-model.ttl \
--input spdx-spec/docs/rdf/jsonld-annotations.ttl \
--context-url spdx-spec/docs/rdf/spdx-context.jsonld https://spdx.org/rdf/3.0.1/spdx-context.jsonld \
jsonschema \
--output spdx-spec/docs/rdf/schema.json
cp spdx-spec/docs/rdf/schema.json spdx-spec/docs/model/schema.json
- name: Set Git identity
working-directory: spdx-spec
run: git config user.name $GIT_USER_NAME; git config user.email $GIT_USER_EMAIL
- name: Sync GitHub Pages
working-directory: spdx-spec
run: git checkout $GH_PAGES_BRANCH && git pull && git checkout $REF_SPEC
- name: Build complete MkDocs configuration
# Combines model file list (MKDOCS_MODEL_YML, generated by spec-parser)
# with the base MkDocs configuration file (MKDOCS_BASE_YML),
# to produce the full MkDocs configuration file (MKDOCS_FULL_YML).
# The script below finds "__MODEL_PLACEHOLDER__" string in
# MKDOCS_BASE_YML, replaces it with the content from MKDOCS_MODEL_YML.
# MKDOCS_FULL_YML will be used by mike in the deploy step.
working-directory: spdx-spec
run: |
echo "Build $MKDOCS_FULL_YML from $MKDOCS_BASE_YML and $MKDOCS_MODEL_YML"
bin/make-mkdocs-config.sh \
-b "$MKDOCS_BASE_YML" \
-m "$MKDOCS_MODEL_YML" \
-f "$MKDOCS_FULL_YML" \
-p "__MODEL_PLACEHOLDER__"
echo "===================="
echo "Full MkDocs configuration: $MKDOCS_FULL_YML"
echo "--------------------"
cat "$MKDOCS_FULL_YML"
echo "===================="
- name: Deploy and set aliases
# mike is used here to manage multiple versions of MkDocs-powered documentation
# This step does 2 things:
# 1) delete existing aliases (in VERSION_ALIASES), if exists
# 2) deploy as VERSION, with aliases
# If the existing aliases were redirected to other versions,
# it means this VERSION will "steal" the aliases from those versions.
working-directory: spdx-spec
run: |
for alias in $VERSION_ALIASES; do
mike delete --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push --allow-empty "$alias" || true
done
mike deploy --update-aliases --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push $VERSION $VERSION_ALIASES
- name: Set default version
# Set default version to VERSION_DEFAULT;
# if not set, the default version will remain the same.
# Should only be done from the "main" branch.
if: github.ref == 'refs/heads/main'
working-directory: spdx-spec
run: |
mike set-default --config-file "$MKDOCS_FULL_YML" --branch $GH_PAGES_BRANCH --push $VERSION_DEFAULT
- name: Copy JSON annotations, JSON schema, JSON-LD context, and RDFs to alias directories
# Fallback for backward compatibility with old URLs before v3.0.1
# This step creates copies of annotations/schema/RDFs to all alias
# directories, so they can be accessible from all old URLs.
# For example,
# - https://spdx.github.io/spdx-spec/v3.0/model/schema.json (old directory structure)
# - https://spdx.github.io/spdx-spec/v3.0.1/rdf/schema.json (new directory structure)
# will all be accessible and have the same content.
# Unlike HTML files, these files have to be a copy,
# since it cannot use the HTML refresh mechanism.
working-directory: spdx-spec
run: |
git checkout $GH_PAGES_BRANCH
dirs="$VERSION_ALIASES"
for dir in $dirs; do
mkdir -p "$dir"/rdf
mkdir -p "$dir"/model
cp $VERSION/rdf/* "$dir"/rdf
cp $VERSION/rdf/* "$dir"/model
mkdir -p "$dir"/diagram
cp $VERSION/diagram/* "$dir"/diagram
mkdir -p "$dir"/jsondump
cp $VERSION/jsondump/* "$dir"/jsondump
git add "$dir"/rdf/* "$dir"/model/* "$dir"/diagram/* "$dir"/jsondump/*
done
git commit -m "Copy schema and RDFs to alias directories: $VERSION_ALIASES"
git push origin $GH_PAGES_BRANCH
- name: Make redirections (for renamed model elements and moved annexes)
# Fallback for backward compatibility with old URLs before v3.0.1
# More redirections can be added in etc/redirect-map.csv (from,to)
# See name changes in model at
# https://github.com/spdx/spdx-3-model/blob/main/CHANGELOG.md
#
# This step creates a HTML files to facilitate additional directions.
# It reads a redirect map from /etc/redirect-map.csv; in the CSV,
# first value is 'from' (source) and second value is 'to' (target).
#
# The 'from' and 'to' values will be inserted into a HTML template at
# from /etc/redirect-template.html, to create a redirect HTML
# page (index.html) under a subdirectory with the name of 'from'
# that will refresh the browser to a URL of 'to'.
#
# For example, given:
#
# VERSION = "v3.0.1"
# VERSION_ALIASES = "latest v3.0"
# from = "model/Core/Properties/imports"
# to = "model/Core/Properties/import"
#
# these HTML files will be created for every aliases:
#
# v3.0.1/model/Core/Properties/imports/index.html
# latest/model/Core/Properties/imports/index.html
# v3.0/model/Core/Properties/imports/index.html
#
# and all of them will redirect to
#
# v3.0.1/model/Core/Properties/import/
working-directory: spdx-spec
run: |
ALL_VERSIONS=$(echo "$VERSION" "$VERSION_ALIASES")
INDEX_HTML="index.html"
git checkout $REF_SPEC
maps=$(cat "$REDIRECT_MAP_PATH")
template=$(cat "$REDIRECT_TEMPLATE_PATH")
echo "===================="
echo "Redirect map: $REDIRECT_MAP_PATH"
echo "--------------------"
echo "$maps"
echo "===================="
echo "===================="
echo "Redirect HTML template: $REDIRECT_TEMPLATE_PATH"
echo "--------------------"
echo "$template"
echo "===================="
git checkout $GH_PAGES_BRANCH
for alias in $ALL_VERSIONS; do
echo "$maps" | while read -r line; do
from=$(echo "$line" | cut -d',' -f1)
to=$(echo "$line" | cut -d',' -f2)
slash_count=$(echo "$from" | tr -cd '/' | wc -c)
upper_dirs=".."
if [ -n "$from" ]; then
for i in $(seq 0 $slash_count); do
upper_dirs="$upper_dirs/.."
done
fi
escaped_upper=$(echo "$upper_dirs" | sed 's/[\/&]/\\&/g')
escaped_version=$(echo "$VERSION" | sed 's/[\/&]/\\&/g')
html=""
case "$to" in
http://*|https://*)
echo "Redirect: $alias/$from -> $to"
escaped_to=$(echo "$to" | sed 's/[\/&]/\\&/g')
html=$(echo "$template" | sed -e "s|__UPPER__/__VERSION__/__TO__|$escaped_to|g")
;;
*)
echo "Redirect: $alias/$from -> $VERSION/$to"
escaped_to=$(echo "$to" | sed 's/[\/&]/\\&/g')
html=$(echo "$template" | sed -e "s/__UPPER__/$escaped_upper/g" -e "s/__VERSION__/$escaped_version/g" -e "s/__TO__/$escaped_to/g")
;;
esac
mkdir -p "$alias/$from"
echo "$html" > "$alias/$from/$INDEX_HTML"
git add "$alias/$from/$INDEX_HTML"
done
done
git commit -m "Add redirections for: $ALL_VERSIONS"
git push origin $GH_PAGES_BRANCH