-
Notifications
You must be signed in to change notification settings - Fork 0
173 lines (147 loc) · 7.22 KB
/
store-api-spec-lint-results.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: Entur/Api/Lint
on:
push: #Remove later. Schedule only runs on default branch
schedule:
- cron: "*/1 * * * *"
jobs:
lint-and-store-result:
name: OpenAPI Lint
runs-on: ubuntu-24.04
environment: dev
permissions:
contents: read
id-token: write
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Checkout guidelines
uses: actions/checkout@v4
with:
repository: entur/api-guidelines
path: rulesets
sparse-checkout: |
.spectral.yml
- name: Authenticate with Google Cloud
id: login-gcp
uses: google-github-actions/[email protected]
with:
workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.SERVICE_ACCOUNT }}
token_format: "access_token"
- name: Check Spec Hashes and Identify Changed Specs
id: check-hashes
shell: bash
run: |
# Initialize an empty array to hold changed spec file paths.
changed_specs=()
#Calulate hash of the linting rules. If the rules change, all specs should be linted.
computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}')
# Check if the /specs directory is empty
if [ -z "$(ls -A ./specs)" ]; then
echo "No spec files found in ./specs. Aborting job."
exit 0 #Variable "changes" is not set to true, so steps below will be ignored.
fi
# Loop over each file in the /specs directory.
for spec in ./specs/*; do
echo "Processing spec file: $spec"
# Compute a SHA256 hash for the spec file.
computed_spec_hash=$(sha256sum "$spec" | awk '{print $1}')
# Extract the service name from the spec.
if [[ "$spec" == *.json ]]; then
service=$(jq -r '.info.title' "$spec")
else
service=$(yq e '.info.title' "$spec")
fi
# Query BigQuery for the latest stored hash for this service.
query="SELECT spec_hash, rules_hash FROM \`ent-apidata-dev.api.rest_api_lint\` WHERE service_name = '$service' ORDER BY created DESC LIMIT 1"
response=$(curl -s -X POST "https://bigquery.googleapis.com/bigquery/v2/projects/ent-apidata-dev/queries" \
--header "Authorization: Bearer ${{ steps.login-gcp.outputs.access_token }}" \
--header "Content-Type: application/json" \
--data "{\"query\": \"$query\", \"useLegacySql\": false}")
stored_spec_hash=$(echo "$response" | jq -r '.rows[0].f[0].v')
stored_rules_hash=$(echo "$response" | jq -r '.rows[0].f[1].v')
# Compare computed hash with the stored hash.
if [ "$computed_spec_hash" != "$stored_spec_hash" ]; then
echo "Spec $spec has changed."
changed_specs+=("$spec")
elif [ "$computed_rules_hash" != "$stored_rules_hash" ]; then
echo "Linting rules have changed. Linting $spec."
changed_specs+=("$spec")
else
echo "Spec $spec is unchanged and linting rules have not changed. Not linting spec."
fi
done
# If no spec file has changed, exit the job.
if [ ${#changed_specs[@]} -eq 0 ]; then
echo "No spec changes detected. Aborting job."
exit 0 #Variable "changes" is not set to true, so steps below will be ignored.
fi
echo "changes=true" >> $GITHUB_OUTPUT
# Convert the array of changed spec files into a space-separated string.
changed_specs_list=$(printf "%s " "${changed_specs[@]}")
# Set an output variable so later steps know which specs to lint.
echo "::set-output name=changed_specs::$changed_specs_list"
- name: Setup Node.js
if: steps.check-hashes.outputs.changes == 'true'
uses: actions/setup-node@v3
with:
node-version: "16"
- name: Install Spectral CLI
if: steps.check-hashes.outputs.changes == 'true'
run: npm install -g @stoplight/spectral-cli
- name: Run Spectral Linting on Changed Specs
if: steps.check-hashes.outputs.changes == 'true'
id: lint
run: |
# Retrieve the list of changed specs from the previous step.
specs=$(echo "${{ steps.check-hashes.outputs.changed_specs }}")
echo "Linting changed specs: $specs"
spectral lint $specs -o lint-result.json --ruleset rulesets/.spectral.yml -f json --quiet || true
- name: Transform lint results into grouped JSON
if: steps.check-hashes.outputs.changes == 'true'
run: |
# Get the spec source file paths from lint-result.json
sources=$(jq -r '.[].source' lint-result.json | sort -u)
# Initialize an empty array for JSON rows.
rows_array=()
# Iterate over each unique source
while IFS= read -r src; do
if [[ "$src" == *.json ]]; then
# If the file is JSON, use jq to extract the service name.
service=$(jq -r '.info.title' "$src")
else
# Otherwise, assume it's YAML and use yq.
service=$(yq e '.info.title' "$src")
fi
# Re-compute the SHA256 hash for the spec file
computed_spec_hash=$(sha256sum "$src" | awk '{print $1}')
# Re-compute the SHA256 hash for the rules file
computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}')
# Get rule violations from lint-result.json for entries with this source and map each to {rule, severity}
results=$(jq --arg src "$src" '[.[] | select(.source == $src) | {rule: .code, severity: .severity}]' lint-result.json)
# Build a row JSON object with the extracted service name and the result array
row=$(jq -n \
--arg service "$service" \
--argjson results "$results" \
--arg spec_hash "$computed_spec_hash" \
--arg rules_hash "$computed_rules_hash" \
'{json: {service_name: $service, lint_result: $results, spec_hash: $spec_hash, rules_hash: $rules_hash}}')
# Append the row to our rows_array
rows_array+=("$row")
done <<< "$sources"
# Combine all rows into the final JSON structure
combined=$(printf '%s\n' "${rows_array[@]}" | jq -s '{rows: .}')
# Write the output to a file
echo "$combined" > transformed.json
- name: Write to BigQuery
if: steps.check-hashes.outputs.changes == 'true'
id: write-to-bigquery
shell: bash
run: |
curl --request POST \
'https://bigquery.googleapis.com/bigquery/v2/projects/ent-apidata-dev/datasets/api/tables/rest_api_lint/insertAll' \
--header "Authorization: Bearer ${{ steps.login-gcp.outputs.access_token }}" \
--header "Accept: application/json" \
--header "Content-Type: application/json" \
--data @transformed.json \
--compressed