diff --git a/.markdown-style.rb b/.markdown-style.rb new file mode 100644 index 0000000..c4366da --- /dev/null +++ b/.markdown-style.rb @@ -0,0 +1,2 @@ +all # Import all rules +exclude_rule "MD013" # Ignore Line length diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 0000000..7cc7d12 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,2 @@ +style "#{File.dirname(__FILE__)}/.markdown-style.rb" +git_recurse true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ecadcac..04a1635 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,6 @@ -exclude: | - (?x) - .drawio$| - ^test/.*.json$| - tsconfig.json$| - .diff$| - .patch$| - .min.| - ^states/common/setup/files/01-netzbegruenung.sh$| - ^states/common/setup/files/01-verdigado.sh$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -40,6 +30,10 @@ repos: rev: v0.9.0.5 hooks: - id: shellcheck + - repo: https://github.com/gitleaks/gitleaks + rev: v8.26.0 + hooks: + - id: gitleaks - repo: local hooks: - id: check-ssh-keys @@ -47,13 +41,7 @@ repos: entry: python build/check-ssh-keys.py language: python files: ^pillars/users/.+\.sls$ - additional_dependencies: ['pyyaml==6.0.1'] # Renovate can't parse it, yet https://github.com/renovatebot/renovate/issues/20780 # TODO - - - id: check-codeowners - name: check CODEOWNERS for alphabetical comment order - entry: python build/check-alphabetical-comments.py - language: python - files: CODEOWNERS + additional_dependencies: ['pyyaml==6.0.2'] # Renovate can't parse it, yet https://github.com/renovatebot/renovate/issues/20780 # TODO - id: prettier # Copied from https://github.com/pre-commit/mirrors-prettier/ instead of referencing it to not rely on their published Prettier versions name: Prettier @@ -65,10 +53,3 @@ repos: require_serial: false additional_dependencies: ['prettier@3'] # Renovate can't parse this, either. Unspecific to prevent local installs, when global installations are available minimum_pre_commit_version: '0' - - - id: git-diff - name: git diff - entry: git diff --exit-code - language: system - pass_filenames: false - always_run: true diff --git a/.woodpecker.yaml b/.woodpecker.yaml deleted file mode 100644 index 463d1a5..0000000 --- a/.woodpecker.yaml +++ /dev/null @@ -1,32 +0,0 @@ -when: - path: '*Dockerfile*' - -steps: - build-main: - when: - branch: main - image: woodpeckerci/plugin-docker-buildx:4.0.0@sha256:9d24b71c37d7a958d79252e608c4d1a04b02f2e74d4e26003b43e0830038bde0 - pull: true - settings: - platforms: linux/amd64 - registry: ${CI_FORGE_URL} - username: WoodpeckerCI - password: - from_secret: gitea_token - repo: git.verdigado.com/${CI_REPO,,} - tag: 'latest' - - build-branch: - when: - branch: - exclude: ['main'] - image: woodpeckerci/plugin-docker-buildx:4.0.0@sha256:9d24b71c37d7a958d79252e608c4d1a04b02f2e74d4e26003b43e0830038bde0 - pull: true - settings: - platforms: linux/amd64 - registry: ${CI_FORGE_URL} - username: WoodpeckerCI - password: - from_secret: gitea_token - repo: git.verdigado.com/${CI_REPO,,} - tag: ${CI_COMMIT_BRANCH} diff --git a/.woodpecker/.build.yaml b/.woodpecker/.build.yaml new file mode 100644 index 0000000..d8ed153 --- /dev/null +++ b/.woodpecker/.build.yaml @@ -0,0 +1,35 @@ +steps: + build main: + when: + - event: push + branch: main + image: woodpeckerci/plugin-docker-buildx:6.0.1@sha256:d75734dc72e21033eb539a7e983acbfea5b84c260b78c07247b74d3067d66742 + pull: true + settings: + platforms: linux/amd64 + registry: ${CI_FORGE_URL} + username: WoodpeckerCI + password: + from_secret: gitea_token + repo: git.verdigado.com/${CI_REPO,,} + tags: + - 'latest' + - ${CI_COMMIT_SHA} + + build branch: + when: + - event: push + branch: + exclude: ['main'] + image: woodpeckerci/plugin-docker-buildx:6.0.1@sha256:d75734dc72e21033eb539a7e983acbfea5b84c260b78c07247b74d3067d66742 + pull: true + settings: + platforms: linux/amd64 + registry: ${CI_FORGE_URL} + username: WoodpeckerCI + password: + from_secret: gitea_token + repo: git.verdigado.com/${CI_REPO,,} + tags: + - ${CI_COMMIT_BRANCH} + - ${CI_COMMIT_SHA} diff --git a/.woodpecker/.test.yaml b/.woodpecker/.test.yaml new file mode 100644 index 0000000..083eaf9 --- /dev/null +++ b/.woodpecker/.test.yaml @@ -0,0 +1,56 @@ +skip_clone: true +when: + - event: push +depends_on: + - build +variables: + - &image 'git.verdigado.com/verdigado-images/container-pre-commit:${CI_COMMIT_SHA}' +steps: + await-image: + image: alpine@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c + environment: + IMAGE: *image + commands: + - apk add --update --no-cache img + - 'while !(( img pull $IMAGE 2>&1 | grep -q "Error: failed to unmount" )) ; do echo "Awaiting image $IMAGE..."; sleep 3; done' + - echo 'found.' + + clone salt: + image: woodpeckerci/plugin-git@sha256:d619f364acd546134a9ba06d29a9cb55156eddfc21f60307f1fdcf4e0b2aafa6 + settings: + remote: https://git.verdigado.com/verdigado-Privileged/Salt.git + path: salt + sha: '' + ref: refs/heads/master + branch: master + + pre-commit salt: + image: *image + depends_on: + - await-image + - clone salt + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - cd salt + - pre-commit run --all-files + + clone rocketchat2matrix: + image: woodpeckerci/plugin-git@sha256:d619f364acd546134a9ba06d29a9cb55156eddfc21f60307f1fdcf4e0b2aafa6 + settings: + remote: https://git.verdigado.com/NB-Public/rocketchat2matrix.git + path: rocketchat2matrix + sha: '' + ref: refs/heads/main + branch: master + + pre-commit rocketchat2matrix: + image: *image + depends_on: + - await-image + - clone rocketchat2matrix + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - cd rocketchat2matrix + - pre-commit run --all-files diff --git a/Dockerfile b/Dockerfile index d5f6e51..c9c2201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,17 @@ -FROM python:3.12.4-alpine3.20@sha256:a982997504b8ec596f553d78f4de4b961bbdf5254e0177f6e99bb34f4ef16f95 -COPY --from=koalaman/shellcheck:v0.10.0@sha256:2097951f02e735b613f4a34de20c40f937a6c8f18ecb170612c88c34517221fb /bin/shellcheck /usr/bin/ -COPY --from=ghcr.io/gitleaks/gitleaks:v8.18.4@sha256:f44e526acc67786b7476db413edb993ce2d152660d32fb3eb48d9bca06fa83f8 /usr/bin/gitleaks /usr/bin/ +FROM python:3.13.3-alpine3.20@sha256:40a4559d3d6b2117b1fbe426f17d55b9100fa40609733a1d0c3f39e2151d4b33 # renovate: datasource=repology depName=alpine_3_20/build-base versioning=loose ENV BUILD_BASE_VERSION="0.5-r3" # renovate: datasource=repology depName=alpine_3_20/gcc versioning=loose -ENV GCC_VERSION="13.2.1_git20240309-r0" +ENV GCC_VERSION="13.2.1_git20240309-r1" # renovate: datasource=repology depName=alpine_3_20/ruby versioning=loose -ENV RUBY_VERSION="3.3.1-r0" +ENV RUBY_VERSION="3.3.8-r0" # renovate: datasource=repology depName=alpine_3_20/git versioning=loose -ENV GIT_VERSION="2.45.2-r0" +ENV GIT_VERSION="2.45.3-r0" # renovate: datasource=repology depName=alpine_3_20/openssh-keygen versioning=loose -ENV OPENSSH_KEYGEN_VERSION="9.7_p1-r3" +ENV OPENSSH_KEYGEN_VERSION="9.7_p1-r5" # renovate: datasource=pypi depName=pre-commit versioning=pep440 -ENV PRE_COMMIT_VERSION="3.7.1" -# renovate: datasource=rubygems depName=mdl versioning=ruby -ENV MDL_VERSION="0.13.0" +ENV PRE_COMMIT_VERSION="4.2.0" RUN mkdir /data /tmp/pre-commit COPY .pre-commit-config.yaml /tmp/pre-commit @@ -23,13 +19,11 @@ COPY .pre-commit-config.yaml /tmp/pre-commit RUN apk add --update --no-cache \ build-base="${BUILD_BASE_VERSION}" \ gcc="${GCC_VERSION}" \ - ruby="${RUBY_VERSION}" \ ruby-dev="${RUBY_VERSION}" \ git="${GIT_VERSION}" \ openssh-keygen="${OPENSSH_KEYGEN_VERSION}" \ && \ pip install --no-cache-dir pre-commit=="${PRE_COMMIT_VERSION}" && \ - gem install --no-document mdl -v "${MDL_VERSION}" && \ git config --global --add safe.directory /data && \ cd /tmp/pre-commit && \ git init --initial-branch main && \ diff --git a/README.md b/README.md new file mode 100644 index 0000000..e193f46 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +# verdigado pre-commit container + +A container image to include all dependencies (and a warmed up cache) used in our [`pre-commit`](https://pre-commit.com/) hooks/CI steps to speed up execution. + +If you see any pre-commit CI jobs installing dependencies: + +- Make sure to execute `pre-commit` using this container +- Add the hook to this repo's `.pre-commit-config.yaml` +- Optionally install dependencies in the `Dockerfile` with the versions set up for `Renovate` + +## Usage + +In your `.woodpecker.yaml`, adapt and add the following block: + +```yaml +steps: + check-pre-commit: + image: git.verdigado.com/verdigado-images/container-pre-commit:latest + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - pre-commit run --all-files +``` + +If renovate is set up for your repo, it'll add and update the pinned digest/hash of the image. + +## Development + +Generally you should have `Docker` or something alike installed. + +If you need to copy files into the container, don't forget to add exclusions to the general _exclude all_ in `.dockerignore`. + +To **update the base image** (like `3.12.4-alpine3.20` to a newer Alpine version), manual work is still required, but supported by a little script. **Renovate might not create a PR for newer image tags.** + +1. In the `Dockerfile`, update the Alpine version for the image and the renovate comments (`# renovate: datasource=repology depName=alpine_3_20/gcc versioning=loose`). + + ```diff + - FROM python:3-alpine3.19@sha256:00c0ffeeacab... + + FROM python:3-alpine3.20 # You can omit the sha256 digest, the script prints it out + # ... + + - # renovate: datasource=repology depName=alpine_3_19/build-base versioning=loose + + # renovate: datasource=repology depName=alpine_3_20/build-base versioning=loose + ENV BUILD_BASE_VERSION="0.8.15" + # ... + ``` + +1. Now run `./get_pkg_versions.sh`. It pulls the alpine image from the Dockerfile, prints it's digest and the latest packages it could find via `apk` inside that container and prints out the names and versions. + + Example output of `./get_pkg_versions.sh` for a new image, which is not yet pulled: + + ```plain + Unable to find image 'python:3.12.3-alpine3.18' locally + 3.12.3-alpine3.18: Pulling from library/python + 619be1103602: Pull complete + [...] + 0eb61f1af52e: Pull complete + Digest: sha256:24680ddf8422899b24756d62b31eb5de782fbb42e9c2bb1c70f1f55fcf891721 + Status: Downloaded newer image for python:3.12.3-alpine3.18 + [Script output starts here] + Checking 5/5 latest package versions on python:3.12.3-alpine3.18 + Image digest found: sha256:24680ddf8422899b24756d62b31eb5de782fbb42e9c2bb1c70f1f55fcf891721 + --- + build-base-0.5-r3 + gcc-12.2.1_git20220924-r10 + git-2.40.1-r0 + openssh-keygen-9.3_p2-r1 + ruby-3.2.4-r0 + ``` + +1. Copy the package versions and update the respective `ENV` with it manually in the `Dockerfile`. You also might add the digest to the base image. + +1. Test building the image and you can commit it. diff --git a/get_pkg_versions.sh b/get_pkg_versions.sh new file mode 100755 index 0000000..87b772f --- /dev/null +++ b/get_pkg_versions.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +IMAGE=$(grep -oP 'FROM \K.*alpine[^ ]+' Dockerfile) +PACKAGES=$(grep -oP '#.+depName=alpine.+/\K[^ ]+' Dockerfile) +# shellcheck disable=SC2086 +PACKAGES_NO_BR=$(echo ${PACKAGES} | tr -d '\n') +PACKAGES_VERSIONS=$(docker run --rm -t --entrypoint /bin/sh "$IMAGE" -c "apk --update --no-cache list $PACKAGES_NO_BR | cut -d ' ' -f 1 | grep -v '^fetch$'") +DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE" | cut -d '@' -f2) + +echo "Checking $(echo "$PACKAGES" | wc -l)/$(echo "$PACKAGES_VERSIONS" | wc -l) latest package versions on $IMAGE" +echo "Image digest found: $DIGEST" +echo "---" +echo "$PACKAGES_VERSIONS" diff --git a/renovate.json b/renovate.json index 2cb753a..f0fc161 100644 --- a/renovate.json +++ b/renovate.json @@ -8,14 +8,14 @@ { "groupName": "all dependencies", "groupSlug": "all", - "matchPackagePatterns": ["*"] + "matchPackageNames": ["/*/"] } ], "separateMajorMinor": false, "customManagers": [ { "customType": "regex", - "fileMatch": ["^Dockerfile$"], + "managerFilePatterns": ["/^Dockerfile$/"], "matchStrings": [ "#\\s*renovate:\\s*datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?\\sENV .*?_VERSION=\"(?.*)\"\\s" ],