mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-02 09:04:51 +02:00
Chromium und Alpine update (#319)
* Update base image to alpine v3.19 * Update Chromium to 121 * Remove libssl1.1 (no longer available in alpine 3.19) * Add --break-system-packages to pip install commands * Print debugging info * Upgrade actions/checkout to v4 * Fix entrypoints and commands * Upgrade pyOpenSSL to v24.0.0 * Upgrade tenacity to v8
This commit is contained in:
parent
a6af33772e
commit
0efab3295c
2
.github/workflows/test.yaml
vendored
2
.github/workflows/test.yaml
vendored
|
@ -6,7 +6,7 @@ jobs:
|
|||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build Docker image
|
||||
run: make dockerimage
|
||||
- name: Create placeholder secrets
|
||||
|
|
10
Dockerfile
10
Dockerfile
|
@ -1,6 +1,6 @@
|
|||
FROM alpine:3.16.2
|
||||
FROM alpine:3.19
|
||||
|
||||
ENV CHROMIUM_VERSION=106.0.5249.119-r1
|
||||
ENV CHROMIUM_VERSION=121.0.6167.184-r0
|
||||
|
||||
RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories && \
|
||||
echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
|
||||
|
@ -8,7 +8,7 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositori
|
|||
chromium=$CHROMIUM_VERSION \
|
||||
chromium-chromedriver=$CHROMIUM_VERSION \
|
||||
py3-cryptography python3-dev py3-grpcio py3-wheel py3-pip py3-lxml py3-yaml \
|
||||
build-base git icu-libs libssl1.1 libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
|
||||
build-base git icu-libs libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
|
||||
libffi-dev openssl-dev cargo
|
||||
|
||||
RUN apk info -v | sort
|
||||
|
@ -16,13 +16,13 @@ RUN apk info -v | sort
|
|||
WORKDIR /workdir
|
||||
|
||||
# Execute time consuming compilations in a separate step
|
||||
RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0
|
||||
RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0 --break-system-packages
|
||||
|
||||
ADD https://pki.google.com/roots.pem /google_roots.pem
|
||||
ENV GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/google_roots.pem
|
||||
|
||||
ADD requirements.txt /workdir/
|
||||
RUN pip install -r requirements.txt
|
||||
RUN pip install -r requirements.txt --break-system-packages
|
||||
|
||||
RUN python3 -m pip freeze
|
||||
|
||||
|
|
17
Makefile
17
Makefile
|
@ -28,18 +28,20 @@ spider:
|
|||
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
|
||||
--shm-size=2g \
|
||||
$(IMAGE) \
|
||||
--credentials-path /secrets/datastore-writer.json \
|
||||
--loglevel debug \
|
||||
spider --kind $(DB_ENTITY) ${ARGS}
|
||||
python3 cli.py \
|
||||
--credentials-path /secrets/datastore-writer.json \
|
||||
--loglevel debug \
|
||||
spider --kind $(DB_ENTITY) ${ARGS}
|
||||
|
||||
export:
|
||||
docker run --rm -ti \
|
||||
-v $(PWD)/secrets:/secrets \
|
||||
-v $(PWD)/volumes/json-export:/json-export \
|
||||
$(IMAGE) \
|
||||
--credentials-path /secrets/datastore-reader.json \
|
||||
--loglevel debug \
|
||||
export --kind $(DB_ENTITY)
|
||||
python3 cli.py \
|
||||
--credentials-path /secrets/datastore-reader.json \
|
||||
--loglevel debug \
|
||||
export --kind $(DB_ENTITY)
|
||||
|
||||
# run spider tests
|
||||
test:
|
||||
|
@ -48,9 +50,8 @@ test:
|
|||
-v $(PWD)/secrets:/secrets \
|
||||
-v $(PWD)/screenshots:/screenshots \
|
||||
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
|
||||
--entrypoint "python3" \
|
||||
$(IMAGE) \
|
||||
-m unittest discover -p '*_test.py' -v
|
||||
python3 -m unittest discover -p '*_test.py' -v
|
||||
|
||||
VERSION:
|
||||
@echo $(VERSION) > VERSION
|
||||
|
|
|
@ -25,7 +25,7 @@ protobuf==4.21.8
|
|||
pyasn1==0.4.8
|
||||
pyasn1-modules==0.2.8
|
||||
pycparser==2.21
|
||||
pyOpenSSL==22.0.0
|
||||
pyOpenSSL==23.0.0
|
||||
pytz==2021.3
|
||||
redis==4.1.0
|
||||
requests==2.26.0
|
||||
|
@ -36,7 +36,7 @@ selenium==3.141.0
|
|||
smmap==3.0.4
|
||||
smmap2==2.0.5
|
||||
soupsieve==2.2.1
|
||||
tenacity==5.1.5
|
||||
tenacity==8.2.3
|
||||
urllib3==1.26.7
|
||||
w3lib==1.22.0
|
||||
websocket-client==0.59.0
|
||||
|
|
Loading…
Reference in a new issue