diff --git a/.dockerignore b/.dockerignore index ea32980..7e91a32 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ # Ignore everything * +!.pre-commit-config.yaml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4bd3180 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +# Non-standard +quote_type = single + +[*.{diff,patch}] +indent_style = unset +indent_size = unset diff --git a/.markdown-style.rb b/.markdown-style.rb new file mode 100644 index 0000000..c4366da --- /dev/null +++ b/.markdown-style.rb @@ -0,0 +1,2 @@ +all # Import all rules +exclude_rule "MD013" # Ignore Line length diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 0000000..7cc7d12 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,2 @@ +style "#{File.dirname(__FILE__)}/.markdown-style.rb" +git_recurse true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..04a1635 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-xml + - id: check-yaml + - id: double-quote-string-fixer + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: no-commit-to-branch + - id: requirements-txt-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/warpnet/salt-lint + rev: v0.9.2 + hooks: + - id: salt-lint + - repo: https://github.com/markdownlint/markdownlint + rev: v0.13.0 + hooks: + - id: markdownlint + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.5 + hooks: + - id: shellcheck + - repo: https://github.com/gitleaks/gitleaks + rev: v8.26.0 + hooks: + - id: gitleaks + - repo: local + hooks: + - id: check-ssh-keys + name: check SSH public keys in user pillars + entry: python build/check-ssh-keys.py + language: python + files: ^pillars/users/.+\.sls$ + additional_dependencies: ['pyyaml==6.0.2'] # Renovate can't parse it, yet https://github.com/renovatebot/renovate/issues/20780 # TODO + + - id: prettier # Copied from https://github.com/pre-commit/mirrors-prettier/ instead of referencing it to not rely on their published Prettier versions + name: Prettier + description: '' + entry: prettier --write --ignore-unknown + language: node + 'types': [text] + args: [] + require_serial: false + additional_dependencies: ['prettier@3'] # Renovate can't parse this, either. Unspecific to prevent local installs, when global installations are available + minimum_pre_commit_version: '0' diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000..85e2c9f --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,4 @@ +semi: false +bracketSpacing: true +trailingComma: es5 +proseWrap: preserve diff --git a/.woodpecker.yaml b/.woodpecker.yaml deleted file mode 100644 index 27af0e5..0000000 --- a/.woodpecker.yaml +++ /dev/null @@ -1,32 +0,0 @@ -when: - path: "*Dockerfile*" - -steps: - build-main: - when: - branch: main - image: woodpeckerci/plugin-docker-buildx:3.2.1@sha256:a4a4e4cfd6ca3a8234d7bc87e771fea0ced1326d174e2620f670331358cbcddb - pull: true - settings: - platforms: linux/amd64 - registry: ${CI_FORGE_URL} - username: WoodpeckerCI - password: - from_secret: gitea_token - repo: git.verdigado.com/${CI_REPO,,} - tag: "latest" - - build-branch: - when: - branch: - exclude: ["main"] - image: woodpeckerci/plugin-docker-buildx:3.2.1@sha256:a4a4e4cfd6ca3a8234d7bc87e771fea0ced1326d174e2620f670331358cbcddb - pull: true - settings: - platforms: linux/amd64 - registry: ${CI_FORGE_URL} - username: WoodpeckerCI - password: - from_secret: gitea_token - repo: git.verdigado.com/${CI_REPO,,} - tag: ${CI_COMMIT_BRANCH} diff --git a/.woodpecker/.build.yaml b/.woodpecker/.build.yaml new file mode 100644 index 0000000..d8ed153 --- /dev/null +++ b/.woodpecker/.build.yaml @@ -0,0 +1,35 @@ +steps: + build main: + when: + - event: push + branch: main + image: woodpeckerci/plugin-docker-buildx:6.0.1@sha256:d75734dc72e21033eb539a7e983acbfea5b84c260b78c07247b74d3067d66742 + pull: true + settings: + platforms: linux/amd64 + registry: ${CI_FORGE_URL} + username: WoodpeckerCI + password: + from_secret: gitea_token + repo: git.verdigado.com/${CI_REPO,,} + tags: + - 'latest' + - ${CI_COMMIT_SHA} + + build branch: + when: + - event: push + branch: + exclude: ['main'] + image: woodpeckerci/plugin-docker-buildx:6.0.1@sha256:d75734dc72e21033eb539a7e983acbfea5b84c260b78c07247b74d3067d66742 + pull: true + settings: + platforms: linux/amd64 + registry: ${CI_FORGE_URL} + username: WoodpeckerCI + password: + from_secret: gitea_token + repo: git.verdigado.com/${CI_REPO,,} + tags: + - ${CI_COMMIT_BRANCH} + - ${CI_COMMIT_SHA} diff --git a/.woodpecker/.test.yaml b/.woodpecker/.test.yaml new file mode 100644 index 0000000..083eaf9 --- /dev/null +++ b/.woodpecker/.test.yaml @@ -0,0 +1,56 @@ +skip_clone: true +when: + - event: push +depends_on: + - build +variables: + - &image 'git.verdigado.com/verdigado-images/container-pre-commit:${CI_COMMIT_SHA}' +steps: + await-image: + image: alpine@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c + environment: + IMAGE: *image + commands: + - apk add --update --no-cache img + - 'while !(( img pull $IMAGE 2>&1 | grep -q "Error: failed to unmount" )) ; do echo "Awaiting image $IMAGE..."; sleep 3; done' + - echo 'found.' + + clone salt: + image: woodpeckerci/plugin-git@sha256:d619f364acd546134a9ba06d29a9cb55156eddfc21f60307f1fdcf4e0b2aafa6 + settings: + remote: https://git.verdigado.com/verdigado-Privileged/Salt.git + path: salt + sha: '' + ref: refs/heads/master + branch: master + + pre-commit salt: + image: *image + depends_on: + - await-image + - clone salt + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - cd salt + - pre-commit run --all-files + + clone rocketchat2matrix: + image: woodpeckerci/plugin-git@sha256:d619f364acd546134a9ba06d29a9cb55156eddfc21f60307f1fdcf4e0b2aafa6 + settings: + remote: https://git.verdigado.com/NB-Public/rocketchat2matrix.git + path: rocketchat2matrix + sha: '' + ref: refs/heads/main + branch: master + + pre-commit rocketchat2matrix: + image: *image + depends_on: + - await-image + - clone rocketchat2matrix + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - cd rocketchat2matrix + - pre-commit run --all-files diff --git a/Dockerfile b/Dockerfile index 365f936..c9c2201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,33 @@ -FROM python:3.12.3-alpine3.19@sha256:ef097620baf1272e38264207003b0982285da3236a20ed829bf6bbf1e85fe3cb -COPY --from=koalaman/shellcheck:v0.10.0@sha256:2097951f02e735b613f4a34de20c40f937a6c8f18ecb170612c88c34517221fb /bin/shellcheck /bin/ +FROM python:3.13.3-alpine3.20@sha256:40a4559d3d6b2117b1fbe426f17d55b9100fa40609733a1d0c3f39e2151d4b33 -# renovate: datasource=repology depName=alpine_3_19/build-base versioning=loose +# renovate: datasource=repology depName=alpine_3_20/build-base versioning=loose ENV BUILD_BASE_VERSION="0.5-r3" -# renovate: datasource=repology depName=alpine_3_19/gcc versioning=loose -ENV GCC_VERSION="13.2.1_git20231014-r0" -# renovate: datasource=repology depName=alpine_3_19/ruby versioning=loose -ENV RUBY_VERSION="3.2.2-r1" -# renovate: datasource=repology depName=alpine_3_19/git versioning=loose -ENV GIT_VERSION="2.43.0-r0" -# renovate: datasource=repology depName=alpine_3_19/openssh-keygen versioning=loose -ENV OPENSSH_KEYGEN_VERSION="9.6_p1-r0" +# renovate: datasource=repology depName=alpine_3_20/gcc versioning=loose +ENV GCC_VERSION="13.2.1_git20240309-r1" +# renovate: datasource=repology depName=alpine_3_20/ruby versioning=loose +ENV RUBY_VERSION="3.3.8-r0" +# renovate: datasource=repology depName=alpine_3_20/git versioning=loose +ENV GIT_VERSION="2.45.3-r0" +# renovate: datasource=repology depName=alpine_3_20/openssh-keygen versioning=loose +ENV OPENSSH_KEYGEN_VERSION="9.7_p1-r5" # renovate: datasource=pypi depName=pre-commit versioning=pep440 -ENV PRE_COMMIT_VERSION="3.7.0" -# renovate: datasource=rubygems depName=mdl versioning=ruby -ENV MDL_VERSION="0.13.0" +ENV PRE_COMMIT_VERSION="4.2.0" + +RUN mkdir /data /tmp/pre-commit +COPY .pre-commit-config.yaml /tmp/pre-commit RUN apk add --update --no-cache \ build-base="${BUILD_BASE_VERSION}" \ gcc="${GCC_VERSION}" \ - ruby="${RUBY_VERSION}" \ + ruby-dev="${RUBY_VERSION}" \ git="${GIT_VERSION}" \ openssh-keygen="${OPENSSH_KEYGEN_VERSION}" \ && \ pip install --no-cache-dir pre-commit=="${PRE_COMMIT_VERSION}" && \ - gem install --no-document mdl -v "${MDL_VERSION}" && \ - mkdir /data && \ - git config --global --add safe.directory /data + git config --global --add safe.directory /data && \ + cd /tmp/pre-commit && \ + git init --initial-branch main && \ + pre-commit install --install-hooks && \ + rm -rf /tmp/pre-commit WORKDIR /data diff --git a/README.md b/README.md new file mode 100644 index 0000000..e193f46 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +# verdigado pre-commit container + +A container image to include all dependencies (and a warmed up cache) used in our [`pre-commit`](https://pre-commit.com/) hooks/CI steps to speed up execution. + +If you see any pre-commit CI jobs installing dependencies: + +- Make sure to execute `pre-commit` using this container +- Add the hook to this repo's `.pre-commit-config.yaml` +- Optionally install dependencies in the `Dockerfile` with the versions set up for `Renovate` + +## Usage + +In your `.woodpecker.yaml`, adapt and add the following block: + +```yaml +steps: + check-pre-commit: + image: git.verdigado.com/verdigado-images/container-pre-commit:latest + environment: + - SKIP=no-commit-to-branch # Ignore "don't commit to protected branch" check + commands: + - pre-commit run --all-files +``` + +If renovate is set up for your repo, it'll add and update the pinned digest/hash of the image. + +## Development + +Generally you should have `Docker` or something alike installed. + +If you need to copy files into the container, don't forget to add exclusions to the general _exclude all_ in `.dockerignore`. + +To **update the base image** (like `3.12.4-alpine3.20` to a newer Alpine version), manual work is still required, but supported by a little script. **Renovate might not create a PR for newer image tags.** + +1. In the `Dockerfile`, update the Alpine version for the image and the renovate comments (`# renovate: datasource=repology depName=alpine_3_20/gcc versioning=loose`). + + ```diff + - FROM python:3-alpine3.19@sha256:00c0ffeeacab... + + FROM python:3-alpine3.20 # You can omit the sha256 digest, the script prints it out + # ... + + - # renovate: datasource=repology depName=alpine_3_19/build-base versioning=loose + + # renovate: datasource=repology depName=alpine_3_20/build-base versioning=loose + ENV BUILD_BASE_VERSION="0.8.15" + # ... + ``` + +1. Now run `./get_pkg_versions.sh`. It pulls the alpine image from the Dockerfile, prints it's digest and the latest packages it could find via `apk` inside that container and prints out the names and versions. + + Example output of `./get_pkg_versions.sh` for a new image, which is not yet pulled: + + ```plain + Unable to find image 'python:3.12.3-alpine3.18' locally + 3.12.3-alpine3.18: Pulling from library/python + 619be1103602: Pull complete + [...] + 0eb61f1af52e: Pull complete + Digest: sha256:24680ddf8422899b24756d62b31eb5de782fbb42e9c2bb1c70f1f55fcf891721 + Status: Downloaded newer image for python:3.12.3-alpine3.18 + [Script output starts here] + Checking 5/5 latest package versions on python:3.12.3-alpine3.18 + Image digest found: sha256:24680ddf8422899b24756d62b31eb5de782fbb42e9c2bb1c70f1f55fcf891721 + --- + build-base-0.5-r3 + gcc-12.2.1_git20220924-r10 + git-2.40.1-r0 + openssh-keygen-9.3_p2-r1 + ruby-3.2.4-r0 + ``` + +1. Copy the package versions and update the respective `ENV` with it manually in the `Dockerfile`. You also might add the digest to the base image. + +1. Test building the image and you can commit it. diff --git a/get_pkg_versions.sh b/get_pkg_versions.sh new file mode 100755 index 0000000..87b772f --- /dev/null +++ b/get_pkg_versions.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +IMAGE=$(grep -oP 'FROM \K.*alpine[^ ]+' Dockerfile) +PACKAGES=$(grep -oP '#.+depName=alpine.+/\K[^ ]+' Dockerfile) +# shellcheck disable=SC2086 +PACKAGES_NO_BR=$(echo ${PACKAGES} | tr -d '\n') +PACKAGES_VERSIONS=$(docker run --rm -t --entrypoint /bin/sh "$IMAGE" -c "apk --update --no-cache list $PACKAGES_NO_BR | cut -d ' ' -f 1 | grep -v '^fetch$'") +DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE" | cut -d '@' -f2) + +echo "Checking $(echo "$PACKAGES" | wc -l)/$(echo "$PACKAGES_VERSIONS" | wc -l) latest package versions on $IMAGE" +echo "Image digest found: $DIGEST" +echo "---" +echo "$PACKAGES_VERSIONS" diff --git a/renovate.json b/renovate.json index 19827ba..f0fc161 100644 --- a/renovate.json +++ b/renovate.json @@ -1,8 +1,6 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "local>renovate/config" - ], + "extends": ["local>renovate/config"], "branchPrefix": "renovate-", "groupName": "all dependencies", "groupSlug": "all", @@ -10,18 +8,14 @@ { "groupName": "all dependencies", "groupSlug": "all", - "matchPackagePatterns": [ - "*" - ] + "matchPackageNames": ["/*/"] } ], "separateMajorMinor": false, "customManagers": [ { "customType": "regex", - "fileMatch": [ - "^Dockerfile$" - ], + "managerFilePatterns": ["/^Dockerfile$/"], "matchStrings": [ "#\\s*renovate:\\s*datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?\\sENV .*?_VERSION=\"(?.*)\"\\s" ],