Chromium und Alpine update (#319)

* Update base image to alpine v3.19

* Update Chromium to 121

* Remove libssl1.1 (no longer available in alpine 3.19)

* Add --break-system-packages to pip install commands

* Print debugging info

* Upgrade actions/checkout to v4

* Fix entrypoints and commands

* Upgrade pyOpenSSL to v24.0.0

* Upgrade tenacity to v8
This commit is contained in:
Marian Steinbach 2024-02-21 09:19:33 +01:00 committed by GitHub
parent a6af33772e
commit 0efab3295c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 17 additions and 16 deletions

View File

@ -6,7 +6,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Build Docker image
run: make dockerimage
- name: Create placeholder secrets

View File

@ -1,6 +1,6 @@
FROM alpine:3.16.2
FROM alpine:3.19
ENV CHROMIUM_VERSION=106.0.5249.119-r1
ENV CHROMIUM_VERSION=121.0.6167.184-r0
RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories && \
echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
@ -8,7 +8,7 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositori
chromium=$CHROMIUM_VERSION \
chromium-chromedriver=$CHROMIUM_VERSION \
py3-cryptography python3-dev py3-grpcio py3-wheel py3-pip py3-lxml py3-yaml \
build-base git icu-libs libssl1.1 libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
build-base git icu-libs libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
libffi-dev openssl-dev cargo
RUN apk info -v | sort
@ -16,13 +16,13 @@ RUN apk info -v | sort
WORKDIR /workdir
# Execute time consuming compilations in a separate step
RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0
RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0 --break-system-packages
ADD https://pki.google.com/roots.pem /google_roots.pem
ENV GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/google_roots.pem
ADD requirements.txt /workdir/
RUN pip install -r requirements.txt
RUN pip install -r requirements.txt --break-system-packages
RUN python3 -m pip freeze

View File

@ -28,18 +28,20 @@ spider:
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
--shm-size=2g \
$(IMAGE) \
--credentials-path /secrets/datastore-writer.json \
--loglevel debug \
spider --kind $(DB_ENTITY) ${ARGS}
python3 cli.py \
--credentials-path /secrets/datastore-writer.json \
--loglevel debug \
spider --kind $(DB_ENTITY) ${ARGS}
export:
docker run --rm -ti \
-v $(PWD)/secrets:/secrets \
-v $(PWD)/volumes/json-export:/json-export \
$(IMAGE) \
--credentials-path /secrets/datastore-reader.json \
--loglevel debug \
export --kind $(DB_ENTITY)
python3 cli.py \
--credentials-path /secrets/datastore-reader.json \
--loglevel debug \
export --kind $(DB_ENTITY)
# run spider tests
test:
@ -48,9 +50,8 @@ test:
-v $(PWD)/secrets:/secrets \
-v $(PWD)/screenshots:/screenshots \
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
--entrypoint "python3" \
$(IMAGE) \
-m unittest discover -p '*_test.py' -v
python3 -m unittest discover -p '*_test.py' -v
VERSION:
@echo $(VERSION) > VERSION

View File

@ -25,7 +25,7 @@ protobuf==4.21.8
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.21
pyOpenSSL==22.0.0
pyOpenSSL==23.0.0
pytz==2021.3
redis==4.1.0
requests==2.26.0
@ -36,7 +36,7 @@ selenium==3.141.0
smmap==3.0.4
smmap2==2.0.5
soupsieve==2.2.1
tenacity==5.1.5
tenacity==8.2.3
urllib3==1.26.7
w3lib==1.22.0
websocket-client==0.59.0