From f2f7989f03753b640366eabe9a0785c824a97801 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 5 Dec 2022 16:11:39 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=94=AF=E6=8C=81python=203.10.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 2 +- docker-compose.yaml | 2 +- feapder_dockerfile | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 5e9b92c..e23d915 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_front:2.5 # 后端镜像 BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_backend:3.7 # 爬虫镜像 -SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4 +SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4_3.10.8 # 监控系统端口配置 INFLUXDB_PORT_TCP=8086 INFLUXDB_PORT_UDP=8089 diff --git a/docker-compose.yaml b/docker-compose.yaml index 77d4b61..90453c2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -37,7 +37,7 @@ services: - SPIDER_AUTO_PULL_IMAGE=1 # 是否自动拉取镜像 否则需要在爬虫节点手动 docker pull 爬虫镜像，为了加快启动速度，可以设置0 - SPIDER_ENV={} # 爬虫环境变量 值为json类型 # 爬虫容器启动参数，支持的参数使用 docker service create --help 查看 - - SPIDER_RUN_ARGS=["--network feaplat", "--log-opt max-size=20m", "--log-opt max-file=1"] + - SPIDER_RUN_ARGS=["--network feaplat", "--log-opt max-size=20m", "--log-opt max-file=1", "--mount type=volume,source=feapder_python3.10,destination=/usr/local/python-3.10.8"] # git ssh 私有密钥，不填则使用默认的 - GIT_SSH_PRIVATE_KEY=${GIT_SSH_PRIVATE_KEY} # 监控配置 diff --git a/feapder_dockerfile b/feapder_dockerfile index 5017d73..7b4ec0b 100644 --- a/feapder_dockerfile +++ b/feapder_dockerfile @@ -1,7 +1,32 @@ -FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:[最新版本号] +FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4 -# 例如安装依赖 -RUN pip3 install feapder \ - && pip3 install scrapy +# 安装自定义的python版本，3.10.8 +RUN set -ex \ + && wget https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz \ + && tar -zxvf Python-3.10.8.tgz \ + && cd Python-3.10.8 \ + && ./configure prefix=/usr/local/python-3.10.8 \ + && make \ + && make install \ + && make clean \ + && rm -rf /Python-3.10.8* \ + # 配置软链接 + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python3.10.8 \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip3.10.8 -# 自己随便搞事情，搞完修改下 `.env`文件里的 SPIDER_IMAGE 的值即可 \ No newline at end of file +# 删除之前的默认python版本 +RUN set -ex \ + && rm -rf /usr/bin/python3 \ + && rm -rf /usr/bin/pip3 \ + && rm -rf /usr/bin/python \ + && rm -rf /usr/bin/pip + +# 设置默认为python3.10.8 +RUN set -ex \ + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python \ + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python3 \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip3 + +# 将python3.10.8加入到环境变量 +ENV PATH=$PATH:/usr/local/python-3.10.8/bin/ \ No newline at end of file From 6d3e037bbe002be255c5a119e09f795f86f8c605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E7=A5=8E=E6=9D=B0?= Date: Thu, 13 Apr 2023 10:44:34 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix=20python3.10.8=20branch=20=E2=80=9CModu?= =?UTF-8?q?leNotFoundError:=20No=20module=20named=20'=5Fssl'=E2=80=9D=20bu?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- feapder_dockerfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/feapder_dockerfile b/feapder_dockerfile index 7b4ec0b..6cf550f 100644 --- a/feapder_dockerfile +++ b/feapder_dockerfile @@ -1,11 +1,27 @@ FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4 +# 安装依赖 +RUN yum update -y && yum install -y openssl openssl-devel + +# 安装自定义的openssl版本，1.1.1 +RUN set -ex \ + && wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz \ + && wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz.sha256 \ + # && sha256sum openssl-1.1.1g.tar.gz \ + && tar -zxvf openssl-1.1.1g.tar.gz \ + && cd openssl-1.1.1g \ + && ./config --prefix=/usr/local/openssl-1.1.1 --openssldir=/usr/local/openssl-1.1.1 no-ssl2 \ + && make \ + && make install \ + && make clean \ + && rm -rf /openssl-1.1.1* + # 安装自定义的python版本，3.10.8 RUN set -ex \ && wget https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz \ && tar -zxvf Python-3.10.8.tgz \ && cd Python-3.10.8 \ - && ./configure prefix=/usr/local/python-3.10.8 \ + && ./configure --prefix=/usr/local/python-3.10.8 --with-openssl=/usr/local/openssl-1.1.1 --with-openssl-rpath=auto \ && make \ && make install \ && make clean \ From df26812b97b69d7f8991d3f148f532f795361a40 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 19 Jun 2023 00:47:14 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpython3.10=20ssl=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 2 +- docker-compose-without-network.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index e23d915..59c4d50 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_front:2.5 # 后端镜像 BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_backend:3.7 # 爬虫镜像 -SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4_3.10.8 +SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.5_3.10.8 # 监控系统端口配置 INFLUXDB_PORT_TCP=8086 INFLUXDB_PORT_UDP=8089 diff --git a/docker-compose-without-network.yaml b/docker-compose-without-network.yaml index af92f02..51cfbcf 100644 --- a/docker-compose-without-network.yaml +++ b/docker-compose-without-network.yaml @@ -37,7 +37,7 @@ services: - SPIDER_AUTO_PULL_IMAGE=1 # 是否自动拉取镜像 否则需要在爬虫节点手动 docker pull 爬虫镜像，为了加快启动速度，可以设置0 - SPIDER_ENV={} # 爬虫环境变量 值为json类型 # 爬虫容器启动参数，支持的参数使用 docker service create --help 查看 - - SPIDER_RUN_ARGS={} + - SPIDER_RUN_ARGS=["--network feaplat", "--log-opt max-size=20m", "--log-opt max-file=1", "--mount type=volume,source=feapder_python3.10,destination=/usr/local/python-3.10.8"] # git ssh 私有密钥，不填则使用默认的 - GIT_SSH_PRIVATE_KEY=${GIT_SSH_PRIVATE_KEY} # 监控配置 From bc1e85757b495b1117076644512c8154023d466e Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 19 Jun 2023 00:49:34 +0800 Subject: [PATCH 4/4] add docker file --- feapder_dockerfile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/feapder_dockerfile b/feapder_dockerfile index 6cf550f..afa166f 100644 --- a/feapder_dockerfile +++ b/feapder_dockerfile @@ -1,13 +1,8 @@ -FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4 - -# 安装依赖 -RUN yum update -y && yum install -y openssl openssl-devel +FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.5 # 安装自定义的openssl版本，1.1.1 RUN set -ex \ - && wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz \ - && wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz.sha256 \ - # && sha256sum openssl-1.1.1g.tar.gz \ + && wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz --no-check-certificate \ && tar -zxvf openssl-1.1.1g.tar.gz \ && cd openssl-1.1.1g \ && ./config --prefix=/usr/local/openssl-1.1.1 --openssldir=/usr/local/openssl-1.1.1 no-ssl2 \ @@ -26,7 +21,6 @@ RUN set -ex \ && make install \ && make clean \ && rm -rf /Python-3.10.8* \ - # 配置软链接 && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python3.10.8 \ && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip3.10.8