From 28c09fdd8772809924f5c58b557ccf42b44d6574 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 18 Sep 2023 21:00:25 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E9=87=8D=E6=96=B0=E6=9E=84=E5=BB=BAworker?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=EF=BC=8Cpython=E5=8D=87=E7=BA=A7=E5=88=B0?= =?UTF-8?q?=E4=BA=863.10=E7=89=88=E6=9C=AC=E3=80=81=E4=B8=94=E6=94=AF?= =?UTF-8?q?=E6=8C=81node16.9.1=E3=80=81java11=EF=BC=8C=E6=9A=82=E6=97=B6?= =?UTF-8?q?=E4=B8=8D=E6=94=AF=E6=8C=81=E6=B5=8F=E8=A7=88=E5=99=A8=E6=B8=B2?= =?UTF-8?q?=E6=9F=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 4 +- docker-compose-without-network.yaml | 122 ---------------------------- docker-compose.yaml | 2 +- 3 files changed, 3 insertions(+), 125 deletions(-) delete mode 100644 docker-compose-without-network.yaml diff --git a/.env b/.env index 88ecd95..c00ef9a 100644 --- a/.env +++ b/.env @@ -11,9 +11,9 @@ BACKEND_WORKER=1 # 前端镜像 FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_front:2.5 # 后端镜像 -BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_backend:3.7 +BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feaplat_backend:3.8 # 爬虫镜像 -SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.5 +SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feaplat_worker:2.6 # 监控系统端口配置 INFLUXDB_PORT_TCP=8086 INFLUXDB_PORT_UDP=8089 diff --git a/docker-compose-without-network.yaml b/docker-compose-without-network.yaml deleted file mode 100644 index af92f02..0000000 --- a/docker-compose-without-network.yaml +++ /dev/null @@ -1,122 +0,0 @@ -version: "3.5" - -x-logging: &default-logging - options: - max-size: "100m" - max-file: "1" - driver: json-file - -services: - feapder_front: - container_name: feapder_front - image: ${FRONT_IMAGE} - restart: always - environment: - - BACKEND_URL=http://feapder_backend:${BACKEND_PORT} - - FRONT_PORT=${FRONT_PORT} - ports: - - ${FRONT_PORT}:${FRONT_PORT} # 前端端口 （自定义端口:80） - depends_on: - - feapder_backend - logging: *default-logging - - feapder_backend: - container_name: feapder_backend - image: ${BACKEND_IMAGE} - restart: always - command: /wait-for-it.sh mysql:3306 -t 60 --strict -- uvicorn main:app --host 0.0.0.0 --workers ${BACKEND_WORKER} --port ${BACKEND_PORT} # workers 为后端服务的个数，爬虫多可改大点 - ports: - - ${BACKEND_PORT}:${BACKEND_PORT} # 后端端口 （自定义端口:8000） - environment: - - FEAPDER_BACKEND_URL=http://${BACKEND_HOST}:${BACKEND_PORT} - - AUTHORIZATION_CODE=${AUTHORIZATION_CODE} # 授权码 - - DB_URL=mysql+pymysql://root:root123@mysql:3306/feapder_platform?charset=utf8mb4 # 后端数据库配置 - - REDIS_DB_URL=redis://:feapderYYDS@redis:6379/0 # redis数据库连接配置 redis://[[username]:[password]]@[host]:[port]/[db] - - ACCESS_TOKEN_EXPIRE_MINUTES=1440 # 管理系统账号cookie过期时间 单位分钟 - - SPIDER_IMAGE=${SPIDER_IMAGE} # 爬虫镜像 - - SPIDER_AUTO_PULL_IMAGE=1 # 是否自动拉取镜像 否则需要在爬虫节点手动 docker pull 爬虫镜像，为了加快启动速度，可以设置0 - - SPIDER_ENV={} # 爬虫环境变量 值为json类型 - # 爬虫容器启动参数，支持的参数使用 docker service create --help 查看 - - SPIDER_RUN_ARGS={} - # git ssh 私有密钥，不填则使用默认的 - - GIT_SSH_PRIVATE_KEY=${GIT_SSH_PRIVATE_KEY} - # 监控配置 - - INFLUXDB_HOST=${INFLUXDB_HOST} - - INFLUXDB_DB=${INFLUXDB_DB} - - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER} - - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD} - - INFLUXDB_PORT_TCP=8086 - - INFLUXDB_PORT_UDP=8089 - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - - "~/data/feapder/projects:/projects" # 上传的项目挂载， 本地目录:容器路径 - depends_on: - - mysql - - redis - - influxdb - logging: *default-logging - - feapder: - container_name: feapder_worker - image: ${SPIDER_IMAGE} - - mysql: - container_name: feapder_backend_mysql - command: mysqld --character-set-server=utf8mb4 - restart: always - image: registry.cn-hangzhou.aliyuncs.com/feapderd/mysql:5.7.29 - ports: - - ${MYSQL_PORT}:3306 - environment: - - "MYSQL_USER=root" - - "MYSQL_ROOT_PASSWORD=root123" - - "MYSQL_DATABASE=feapder_platform" - - "MYSQL_PASSWORD=root123" - volumes: - - "~/data/feapder/feapder_backend_mysql:/var/lib/mysql" # mysql 挂载， 本地目录:容器路径 - logging: *default-logging - - redis: - container_name: feapder_backend_redis - command: redis-server --appendonly yes --requirepass feapderYYDS - restart: always - image: registry.cn-hangzhou.aliyuncs.com/feapderd/redis:6.0.10 -# ports: -# - ${REDIS_PORT}:6379 - volumes: - - "~/data/feapder/feapder_backend_redis:/data" # redis 挂载， 本地目录:容器路径 - logging: *default-logging - -# # 监控 -# grafana: -# user: root -# restart: always -# image: grafana/grafana -# container_name: feapder_grafana -# volumes: -# - ~/data/feapder/grafana:/var/lib/grafana -# environment: -# - GF_SECURITY_ADMIN_USER=admin -# - GF_SECURITY_ADMIN_PASSWORD=admin -# ports: -# - ${GRAFANA_PORT}:3000 -# depends_on: -# - influxdb - - influxdb: - restart: always - image: registry.cn-hangzhou.aliyuncs.com/feapderd/influxdb:1.8.6 - container_name: feapder_influxdb - volumes: - - ~/data/feapder/influxdb:/var/lib/influxdb - - ./conf/influxdb.conf:/etc/influxdb/influxdb.conf - environment: - - INFLUXDB_DB=${INFLUXDB_DB} - - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER} - - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD} - - INFLUXDB_USER=influx - - INFLUXDB_USER_PASSWORD=influx - ports: - - ${INFLUXDB_PORT_TCP}:8086 - - ${INFLUXDB_PORT_UDP}:8089/udp - logging: *default-logging \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 77d4b61..570ed1a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -37,7 +37,7 @@ services: - SPIDER_AUTO_PULL_IMAGE=1 # 是否自动拉取镜像 否则需要在爬虫节点手动 docker pull 爬虫镜像，为了加快启动速度，可以设置0 - SPIDER_ENV={} # 爬虫环境变量 值为json类型 # 爬虫容器启动参数，支持的参数使用 docker service create --help 查看 - - SPIDER_RUN_ARGS=["--network feaplat", "--log-opt max-size=20m", "--log-opt max-file=1"] + - SPIDER_RUN_ARGS=["--network feaplat", "--log-opt max-size=20m", "--log-opt max-file=1", "--mount type=volume,source=feaplat_worker_python3.10.8,destination=/usr/local/python-3.10.8"] # git ssh 私有密钥，不填则使用默认的 - GIT_SSH_PRIVATE_KEY=${GIT_SSH_PRIVATE_KEY} # 监控配置 From a9955c0cfd905ae177bc297ad12264c58ca792ec Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 18 Sep 2023 21:20:31 +0800 Subject: [PATCH 2/8] update feaplat doc --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38f891e..adf78be 100644 --- a/README.md +++ b/README.md @@ -120,15 +120,19 @@ yum -y install git ``` #### 1. 下载项目 +> 先按照下面命令拉取develop分支代码运行。 +> master分支不支持urllib3>=2.0版本，现在已经运行不起来了，但之前老用户不受影响。待后续测试好兼容性，不影响老用户后，会将develop分支合并到master + gitub ```shell -git clone https://github.com/Boris-code/feaplat.git +git clone -b develop https://github.com/Boris-code/feaplat.git ``` gitee ```shell -git clone https://gitee.com/Boris-code/feaplat.git +git clone -b develop https://gitee.com/Boris-code/feaplat.git ``` + #### 2. 运行 首次运行需拉取镜像，时间比较久，且运行可能会报错，再次运行下就好了 From 3a0f62edb276558ab181517762c24fad62eaa761 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Wed, 11 Oct 2023 10:58:35 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcron=E8=A1=A8=E8=BE=BE?= =?UTF-8?q?=E5=BC=8F=E9=AA=8C=E8=AF=81bug=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=E8=B0=83=E5=BA=A6=E5=8F=8A=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index c00ef9a..7c94768 100644 --- a/.env +++ b/.env @@ -11,7 +11,7 @@ BACKEND_WORKER=1 # 前端镜像 FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feapder_front:2.5 # 后端镜像 -BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feaplat_backend:3.8 +BACKEND_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feaplat_backend:3.9 # 爬虫镜像 SPIDER_IMAGE=registry.cn-hangzhou.aliyuncs.com/feapderd/feaplat_worker:2.6 # 监控系统端口配置 From 47bf70593b34002ace9319dbfb9d37c4e9ae964c Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Wed, 28 Feb 2024 14:14:47 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index adf78be..fa55d2f 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ git clone -b develop https://gitee.com/Boris-code/feaplat.git ```shell cd feaplat +docker compose up -d +如果报错，尝试用下面命令运行 docker-compose up -d ``` From 8864508de7c1a095a644874e890c82b3a94f205e Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Wed, 10 Apr 2024 16:13:22 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=A4=9A=E4=BA=86=20network=E8=BF=9E?= =?UTF-8?q?=E4=B8=8D=E4=B8=8A=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 570ed1a..2c49e0a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -126,3 +126,6 @@ networks: name: feaplat driver: overlay attachable: true + ipam: + config: + - subnet: 10.0.0.0/8 From d1c28c484c43238872011d538f044260f4461015 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 15 Apr 2024 17:59:07 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AD=90=E7=BD=91?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 2c49e0a..c7cb9ff 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -128,4 +128,5 @@ networks: attachable: true ipam: config: - - subnet: 10.0.0.0/8 + - subnet: 11.0.0.0/8 + gateway: 11.0.0.1 From 2b2a3861150ebb9dbbc286ad0dfedacb13db3f02 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Fri, 19 Apr 2024 11:01:52 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c7cb9ff..542f26d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -45,8 +45,8 @@ services: - INFLUXDB_DB=${INFLUXDB_DB} - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER} - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD} - - INFLUXDB_PORT_TCP=8086 - - INFLUXDB_PORT_UDP=8089 + - INFLUXDB_PORT_TCP=${INFLUXDB_PORT_TCP} + - INFLUXDB_PORT_UDP=${INFLUXDB_PORT_UDP} volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "~/data/feapder/projects:/projects" # 上传的项目挂载， 本地目录:容器路径 From f8c2f790f86b4097c2b09cb8a41cd23aa79ab406 Mon Sep 17 00:00:00 2001 From: Boris <564773807@qq.com> Date: Mon, 2 Dec 2024 11:27:04 +0800 Subject: [PATCH 8/8] add verify host --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 542f26d..d5bea02 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -28,6 +28,7 @@ services: ports: - ${BACKEND_PORT}:${BACKEND_PORT} # 后端端口 （自定义端口:8000） environment: + - VERIFY_HOST=https://plat.feapder.com - FEAPDER_BACKEND_URL=http://${BACKEND_HOST}:${BACKEND_PORT} - AUTHORIZATION_CODE=${AUTHORIZATION_CODE} # 授权码 - DB_URL=mysql+pymysql://root:root123@mysql:3306/feapder_platform?charset=utf8mb4 # 后端数据库配置