forked from Boris-code/feapder
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcreate_builder.py
More file actions
118 lines (100 loc) · 3.54 KB
/
create_builder.py
File metadata and controls
118 lines (100 loc) · 3.54 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# -*- coding: utf-8 -*-
"""
Created on 2021/2/8 11:21 上午
---------
@summary: 生成器
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
import argparse
import feapder.setting as setting
from feapder.commands.create import *
def main():
spider = argparse.ArgumentParser(description="生成器")
spider.add_argument(
"-p", "--project", help="创建项目 如 feapder create -p <project_name>", metavar=""
)
spider.add_argument(
"-s",
"--spider",
nargs="+",
help="创建爬虫\n"
"如 feapder create -s <spider_name> <spider_type> "
"spider_type=1 AirSpider; "
"spider_type=2 Spider; "
"spider_type=3 BatchSpider;",
metavar="",
)
spider.add_argument(
"-i",
"--item",
nargs="+",
help="创建item 如 feapder create -i test 则生成test表对应的item。 "
"支持like语法模糊匹配所要生产的表。 "
"若想生成支持字典方式赋值的item,则create -item test 1",
metavar="",
)
spider.add_argument(
"-t", "--table", help="根据json创建表 如 feapder create -t <table_name>", metavar=""
)
spider.add_argument(
"-init", help="创建__init__.py 如 feapder create -init", action="store_true"
)
spider.add_argument("-j", "--json", help="创建json", action="store_true")
spider.add_argument("-sj", "--sort_json", help="创建有序json", action="store_true")
spider.add_argument("-c", "--cookies", help="创建cookie", action="store_true")
spider.add_argument("--params", help="解析地址中的参数", action="store_true")
spider.add_argument(
"--setting", help="创建全局配置文件" "feapder create --setting", action="store_true"
)
# 指定数据库
spider.add_argument("--host", type=str, help="mysql 连接地址", metavar="")
spider.add_argument("--port", type=str, help="mysql 端口", metavar="")
spider.add_argument("--username", type=str, help="mysql 用户名", metavar="")
spider.add_argument("--password", type=str, help="mysql 密码", metavar="")
spider.add_argument("--db", type=str, help="mysql 数据库名", metavar="")
args = spider.parse_args()
if args.host:
setting.MYSQL_IP = args.host
if args.port:
setting.MYSQL_PORT = int(args.port)
if args.username:
setting.MYSQL_USER_NAME = args.username
if args.password:
setting.MYSQL_USER_PASS = args.password
if args.db:
setting.MYSQL_DB = args.db
if args.item:
item_name, *support_dict = args.item
support_dict = bool(support_dict)
CreateItem().create(item_name, support_dict)
elif args.spider:
spider_name, *spider_type = args.spider
if not spider_type:
spider_type = 1
else:
spider_type = spider_type[0]
try:
spider_type = int(spider_type)
except:
raise ValueError("spider_type error, support 1, 2, 3")
CreateSpider().create(spider_name, spider_type)
elif args.project:
CreateProject().create(args.project)
elif args.table:
CreateTable().create(args.table)
elif args.init:
CreateInit().create()
elif args.json:
CreateJson().create()
elif args.sort_json:
CreateJson().create(sort_keys=True)
elif args.cookies:
CreateCookies().create()
elif args.setting:
CreateSetting().create()
elif args.params:
CreateParams().create()
if __name__ == "__main__":
main()