From 5ee6ce3eed397e7556e7c098049da5eb44f36b44 Mon Sep 17 00:00:00 2001 From: vanwhebin Date: Wed, 31 Mar 2021 21:51:44 +0800 Subject: [PATCH 1/4] Initial commit --- .gitignore | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 131 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..71964d4 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# python-multithread +python实现多线程的线程池和异步任务调用 From 055800b1af681a6fe7a0f1511f39569ea2d44793 Mon Sep 17 00:00:00 2001 From: vanwhebin Date: Wed, 31 Mar 2021 21:57:13 +0800 Subject: [PATCH 2/4] update ignore files --- .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/multiThreadingPool.iml | 13 ++ .idea/vcs.xml | 6 + .idea/workspace.xml | 275 +++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/multiThreadingPool.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..af37d28 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..61c07f5 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/multiThreadingPool.iml b/.idea/multiThreadingPool.iml new file mode 100644 index 0000000..85c7612 --- /dev/null +++ b/.idea/multiThreadingPool.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..0897334 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,275 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1617107760010 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 3eb8aeedcb2dc84048ee5038dea07db012736253 Mon Sep 17 00:00:00 2001 From: vanwhebin Date: Wed, 7 Apr 2021 18:25:29 +0800 Subject: [PATCH 4/4] parse network headers --- computerNetwork/.gitignore | 4 + computerNetwork/__init__.py | 0 computerNetwork/processor/net/__init__.py | 0 computerNetwork/processor/net/parser.py | 51 +++++++++ computerNetwork/processor/trans/__init__.py | 0 computerNetwork/processor/trans/parser.py | 119 ++++++++++++++++++++ computerNetwork/server.py | 63 +++++++++++ computerNetwork/struct_way.py | 17 +++ 8 files changed, 254 insertions(+) create mode 100644 computerNetwork/.gitignore create mode 100644 computerNetwork/__init__.py create mode 100644 computerNetwork/processor/net/__init__.py create mode 100644 computerNetwork/processor/net/parser.py create mode 100644 computerNetwork/processor/trans/__init__.py create mode 100644 computerNetwork/processor/trans/parser.py create mode 100644 computerNetwork/server.py create mode 100644 computerNetwork/struct_way.py diff --git a/computerNetwork/.gitignore b/computerNetwork/.gitignore new file mode 100644 index 0000000..91591e2 --- /dev/null +++ b/computerNetwork/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +.gitignore +.idea/* +.idea/workspace.xml \ No newline at end of file diff --git a/computerNetwork/__init__.py b/computerNetwork/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/computerNetwork/processor/net/__init__.py b/computerNetwork/processor/net/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/computerNetwork/processor/net/parser.py b/computerNetwork/processor/net/parser.py new file mode 100644 index 0000000..2ab3aef --- /dev/null +++ b/computerNetwork/processor/net/parser.py @@ -0,0 +1,51 @@ +# _*_ encoding=utf-8 _*_ + +import struct +import socket + + +class IPParser: + IP_HEADER_LENGTH = 20 + + @classmethod + def parse_ip_header(cls, ip_header): + """ + IP 报文格式 + line 1: 4位IP_version 4位ip头长度 16位服务类型 + line 2: 16位标识符 3位标记位 3位片位移 + line 3: 8位ttl 8位协议 16位IP头校验和 + line 4: 32位源IP地址 + line 5: 32位目的IP地址 + :param ip_header: + :return: + """ + line1 = struct.unpack(">BBH", ip_header[:4]) + # 11110000 => 1111 + ip_version = line1[0] >> 4 + # 11111111 => 00001111 & 00001111 + iph_length = line1[0] & 15 * 4 + pkg_length = line1[2] + line3 = struct.unpack(">BBH", ip_header[8:12]) + ttl = line3[0] + protocal = line3[1] + iph_checksum = line3[2] + line4 = struct.unpack(">4s", ip_header[12:16]) # 转化位4个arscII码 + src_ip = socket.inet_ntoa(line4[0]) + line5 = struct.unpack(">4s", ip_header[16:20]) # 转化位4个arscII码 + dst_ip = socket.inet_ntoa(line5[0]) + + return { + "ip_version": ip_version, + "iph_length": iph_length, + "packet_length": pkg_length, + "ttl": ttl, + "protocol": protocal, + "iph_checksum": iph_checksum, + "src_ip": src_ip, + "dst_ip": dst_ip, + } + + @classmethod + def parse(cls, ip_header): + return cls.parse_ip_header(ip_header) + diff --git a/computerNetwork/processor/trans/__init__.py b/computerNetwork/processor/trans/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/computerNetwork/processor/trans/parser.py b/computerNetwork/processor/trans/parser.py new file mode 100644 index 0000000..43bc46c --- /dev/null +++ b/computerNetwork/processor/trans/parser.py @@ -0,0 +1,119 @@ + +import struct + + +class TransParser: + + IP_HEADER_OFFSET = 20 + UDP_HEADER_LENGTH = 8 + TCP_HEADER_LENGTH = 20 + + +def data2str(data): + l = len(data) + data - struct.unpack(l*"B", data) + string = "" + for ch in data: + if ch >= 127 or ch < 32: + string += '.' + else: + string += chr(ch) + + return string + + +class UDPParser(TransParser): + """ + line 1: 16位源端口 16位目的端口 + line 2: 16位UDP长度 16位UDP校验和 + """ + + @classmethod + def parse_udp_header(cls, udp_header): + udp_header = struct.unpack(">HHHH", udp_header) + + return { + "src_port": udp_header[0], + "dst_port": udp_header[1], + "udp_length": udp_header[2], + "udp_checksum": udp_header[3], + } + + @classmethod + def parse(cls, packet): + header_offset = cls.IP_HEADER_OFFSET + cls.UDP_HEADER_LENGTH + udp_header = packet[cls.IP_HEADER_OFFSET: header_offset] + data = packet[header_offset:] + result = cls.parse_udp_header(udp_header) + result['data'] = data2str(data) + return result + + +class TCPParser(TransParser): + + @classmethod + def parse_tcp_header(cls, tcp_header): + """ + TCP header 结构 + line 1： 16位源端口 16位目的端口 + line 1： 序列号 + line 1： 确认号 + line 1： 4位数据偏移 6位保留字段 6位标志位 窗口大小 + line 1： 校验和 紧急指针 + :param tcp_header: + :return: + """ + + line1 = struct.unpack(">HH", tcp_header[:4]) + src_port = line1[0] + dst_port = line1[1] + + line2 = struct.unpack(">L", tcp_header[4:8]) + seq_num = line2[0] + + line3 = struct.unpack(">L", tcp_header[8:12]) + ack_num = line3[0] + + line4 = struct.unpack(">BBH", tcp_header[12:16]) + data_offset = line4[0] >> 4 + flags = line4[1] & int('001111111', 2) + FIN = flags & 1 + SYN = (flags >> 1) & 1 + RST = (flags >> 2) & 1 + PSH = (flags >> 3) & 1 + ACK = (flags >> 4) & 1 + URG = (flags >> 5) & 1 + + win_size = line4[2] + + line5 = struct.unpack(">HH", tcp_header[16:20]) + checksum = line5[0] + urg_index = line5[1] + + return { + "src_port": src_port, + "dst_port": dst_port, + "seq_num": seq_num, + "ack_num": ack_num, + "data_offset": data_offset, + "flag": { + "FIN": FIN, + "SYN": SYN, + "RST": RST, + "PSH": PSH, + "ACK": ACK, + "URG": URG + }, + "win_size": win_size, + "checksum": checksum, + "urg_index": urg_index + } + + @classmethod + def parse(cls, packet): + header_offset = cls.IP_HEADER_OFFSET + cls.TCP_HEADER_LENGTH + tcp_header = packet[cls.IP_HEADER_OFFSET: header_offset] + data = packet[header_offset:] + result = cls.parse_tcp_header(tcp_header) + result['data'] = data2str(data) + return result diff --git a/computerNetwork/server.py b/computerNetwork/server.py new file mode 100644 index 0000000..aff4e3c --- /dev/null +++ b/computerNetwork/server.py @@ -0,0 +1,63 @@ + +import json +import socket +from multiThreading.pool import ThreadPool as TP +from multiThreading.task import AsyncTask +from computerNetwork.processor.net.parser import IPParser +from computerNetwork.processor.trans.parser import UDPParser, TCPParser + + +class ProcessTask(AsyncTask): + + def __init__(self, packet, *args, **kwargs): + self.packet = packet + super(AsyncTask, self).__init__(func=self.process, *args, **kwargs) + + def process(self): + headers = { + "network_header": None, + "transport_header": None, + } + ip_header = IPParser.parse(self.packet) + if ip_header['protocol'] == 17: + # UDP头部解析 + ip_header['transport_header'] = UDPParser.parse(self.packet) + elif ip_header['protocol'] == 6: + # TCP头部解析 + ip_header['transport_header'] = TCPParser.parse(self.packet) + return ip_header + + +class Server: + + def __init__(self): + # 工作协议类型IPv4。 套接字类型， 工作具体的协议IP协议 + self.sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_IP) + self.ip = "192.168.0.103" + self.port = 6666 + self.sock.bind((self.ip, self.port),) + + # 混杂模式 + + self.sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON) + + self.pool = TP(10) + TP.start() + + def loop_serve(self): + while True: + # 接收 + packet, address = self.sock.recvfrom(65535) + # 生成task + task = ProcessTask(packet) + # 投入线程池处理 + self.pool.put(task) + # 获取结果 + result = task.get_result() + result = json.dumps(result, indent=4) + print(result) + + +if __name__ == "__main__": + server = Server() + server.loop_serve() diff --git a/computerNetwork/struct_way.py b/computerNetwork/struct_way.py new file mode 100644 index 0000000..51c8740 --- /dev/null +++ b/computerNetwork/struct_way.py @@ -0,0 +1,17 @@ +import struct + +# 八个字节 + +bin_str = b"ABCDEF12" + +print(bin_str) +result = struct.unpack('>BBBBBBBB', bin_str) +print(result) +result = struct.unpack('>HHHH', bin_str) +print(result) +result = struct.unpack('>LL', bin_str) +print(result) +result = struct.unpack('>BBHL', bin_str) +print(result) +result = struct.unpack('>8s', bin_str) +print(result)