#!/usr/bin/env python

# coding:utf-8

# Based on GAppProxy 2.0.0 by Du XiaoGang <dugang.2008@gmail.com>

# Based on WallProxy 0.4.0 by Hust Moon <www.ehust@gmail.com>

# Contributor:

# Phus Lu <phus.lu@gmail.com>

# Hewig Xu <hewigovens@gmail.com>

# Ayanamist Yang <ayanamist@gmail.com>

# V.E.O <V.E.O@tom.com>

# Max Lv <max.c.lv@gmail.com>

# AlsoTang <alsotang@gmail.com>

# Christopher Meng <i@cicku.me>

# Yonsm Guo <YonsmGuo@gmail.com>

# Parkman <cseparkman@gmail.com>

# Ming Bai <mbbill@gmail.com>

# Bin Yu <yubinlove1991@gmail.com>

# lileixuan <lileixuan@gmail.com>

# Cong Ding <cong@cding.org>

# Zhang Youfu <zhangyoufu@gmail.com>

# Lu Wei <luwei@barfoo>

# Harmony Meow <harmony.meow@gmail.com>

# logostream <logostream@gmail.com>

# Rui Wang <isnowfy@gmail.com>

# Wang Wei Qiang <wwqgtxx@gmail.com>

# Felix Yan <felixonmars@gmail.com>

# Sui Feng <suifeng.me@qq.com>

# QXO <qxodream@gmail.com>

# Geek An <geekan@foxmail.com>

# Poly Rabbit <mcx_221@foxmail.com>

# oxnz <yunxinyi@gmail.com>

# Shusen Liu <liushusen.smart@gmail.com>

# Yad Smood <y.s.inside@gmail.com>

# Chen Shuang <cs0x7f@gmail.com>

# cnfuyu <cnfuyu@gmail.com>

# cuixin <steven.cuixin@gmail.com>

# s2marine0 <s2marine0@gmail.com>

# Toshio Xiang <snachx@gmail.com>

# Bo Tian <dxmtb@163.com>

# Virgil <variousvirgil@gmail.com>

# hub01 <miaojiabumiao@yeah.net>

# v3aqb <sgzz.cj@gmail.com>

# Oling Cat <olingcat@gmail.com>

# Meng Zhuo <mengzhuo1203@gmail.com>

# zwhfly <zwhfly@163.com>

# Hubertzhang <hubert.zyk@gmail.com>

# arrix <arrixzhou@gmail.com>

# gwjwin <gwjwin@sina.com>

# Jobin <1149225004@qq.com>

# Zhuhao Wang <zhuhaow@gmail.com>

# YFdyh000 <yfdyh000@gmail.com>

# zzq1015 <zzq1015@users.noreply.github.com>

# Zhengfa Dang <zfdang@users.noreply.github.com>

# haosdent <haosdent@gmail.com>

# xk liu <lxk1012@gmail.com>

__version__ = '3.2.3'

import os

import sys

import sysconfig

import platform

#reload(sys).setdefaultencoding('UTF-8')

sys.dont_write_bytecode = True

current_path = os.path.dirname(os.path.abspath(__file__))

root_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir))

python_path = os.path.join(root_path, 'python27', '1.0')

noarch_lib = os.path.abspath( os.path.join(python_path, 'lib', 'noarch'))

sys.path.append(noarch_lib)

if sys.platform == "win32":

win32_lib = os.path.abspath( os.path.join(python_path, 'lib', 'win32'))

sys.path.append(win32_lib)

elif sys.platform == "linux" or sys.platform == "linux2":

linux_lib = os.path.abspath( os.path.join(python_path, 'lib', 'linux'))

sys.path.append(linux_lib)

elif sys.platform == "darwin":

darwin_lib = os.path.abspath( os.path.join(python_path, 'lib', 'darwin'))

sys.path.append(darwin_lib)

extra_lib = "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python"

sys.path.append(extra_lib)

from cert_util import CertUtil

try:

__import__('gevent.monkey', fromlist=['.']).patch_all()

except (ImportError, SystemError) as e:

print "import gevent fail:", e

sys.exit(sys.stderr.write('please install python-gevent\n'))

import base64

import collections

import ConfigParser

import errno

import httplib

import io

import Queue

import random

import re

import socket

import ssl

import struct

import thread

import threading

import time

import urllib2

import urlparse

import gevent

import OpenSSL

NetWorkIOError = (socket.error, ssl.SSLError, OpenSSL.SSL.Error, OSError)

import logging

from proxylib import AuthFilter

from proxylib import AutoRangeFilter

from proxylib import BaseFetchPlugin

from proxylib import BaseProxyHandlerFilter

from proxylib import BlackholeFilter

from proxylib import CipherFileObject

from proxylib import deflate

from proxylib import DirectFetchPlugin

from proxylib import DirectRegionFilter

from proxylib import dnslib_record2iplist

from proxylib import dnslib_resolve_over_tcp

from proxylib import dnslib_resolve_over_udp

from proxylib import FakeHttpsFilter

from proxylib import ForceHttpsFilter

from proxylib import CRLFSitesFilter

from proxylib import get_dnsserver_list

from proxylib import get_uptime

from proxylib import inflate

from proxylib import LocalProxyServer

from proxylib import message_html

from proxylib import MockFetchPlugin

from proxylib import MultipleConnectionMixin

from proxylib import openssl_set_session_cache_mode

from proxylib import ProxyConnectionMixin

from proxylib import ProxyUtil

from proxylib import RC4Cipher

from proxylib import SimpleProxyHandler

from proxylib import spawn_later

from proxylib import SSLConnection

from proxylib import StaticFileFilter

from proxylib import StripPlugin

from proxylib import StripPluginEx

from proxylib import URLRewriteFilter

from proxylib import UserAgentFilter

from proxylib import XORCipher

import web_control

def is_google_ip(ipaddr):

if ipaddr in ('74.125.127.102', '74.125.155.102', '74.125.39.102', '74.125.39.113', '209.85.229.138'):

return False

if ipaddr.startswith(('173.194.', '207.126.', '209.85.', '216.239.', '64.18.', '64.233.', '66.102.', '66.249.', '72.14.', '74.125.')):

return True

return False

class RangeFetch(object):

"""Range Fetch Class"""

threads = 2

maxsize = 1024*1024*4

bufsize = 8192

waitsize = 1024*512

def __init__(self, handler, plugin, response, fetchservers, **kwargs):

assert isinstance(plugin, BaseFetchPlugin) and hasattr(plugin, 'fetch')

self.handler = handler

self.url = handler.path

self.plugin = plugin

self.response = response

self.fetchservers = fetchservers

self.kwargs = kwargs

self._stopped = None

self._last_app_status = {}

self.expect_begin = 0

def fetch(self):

response_status = self.response.status

response_headers = dict((k.title(), v) for k, v in self.response.getheaders())

content_range = response_headers['Content-Range']

#content_length = response_headers['Content-Length']

start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))

if start == 0:

response_status = 200

response_headers['Content-Length'] = str(length)

del response_headers['Content-Range']

else:

response_headers['Content-Range'] = 'bytes %s-%s/%s' % (start, end, length)

response_headers['Content-Length'] = str(length-start)

logging.info('>>>>>>>>>>>>>>> RangeFetch started(%r) %d-%d', self.url, start, end)

self.handler.send_response(response_status)

for key, value in response_headers.items():

self.handler.send_header(key, value)

self.handler.end_headers()

data_queue = Queue.PriorityQueue()

range_queue = Queue.PriorityQueue()

range_queue.put((start, end, self.response))

self.expect_begin = start

for begin in range(end+1, length, self.maxsize):

range_queue.put((begin, min(begin+self.maxsize-1, length-1), None))

for i in xrange(0, self.threads):

range_delay_size = i * self.maxsize

spawn_later(float(range_delay_size)/self.waitsize, self.__fetchlet, range_queue, data_queue, range_delay_size)

has_peek = hasattr(data_queue, 'peek')

peek_timeout = 120

while self.expect_begin < length - 1:

try:

if has_peek:

begin, data = data_queue.peek(timeout=peek_timeout)

if self.expect_begin == begin:

data_queue.get()

elif self.expect_begin < begin:

time.sleep(0.1)

continue

else:

logging.error('RangeFetch Error: begin(%r) < expect_begin(%r), quit.', begin, self.expect_begin)

break

else:

begin, data = data_queue.get(timeout=peek_timeout)

if self.expect_begin == begin:

pass

elif self.expect_begin < begin:

data_queue.put((begin, data))

time.sleep(0.1)

continue

else:

logging.error('RangeFetch Error: begin(%r) < expect_begin(%r), quit.', begin, self.expect_begin)

break

except Queue.Empty:

logging.error('data_queue peek timeout, break')

break

try:

self.handler.wfile.write(data)

self.expect_begin += len(data)

del data

except Exception as e:

logging.info('RangeFetch client connection aborted(%s).', e)

break

self._stopped = True

def __fetchlet(self, range_queue, data_queue, range_delay_size):

headers = dict((k.title(), v) for k, v in self.handler.headers.items())

headers['Connection'] = 'close'

while 1:

try:

if self._stopped:

return

try:

start, end, response = range_queue.get(timeout=1)

if self.expect_begin < start and data_queue.qsize() * self.bufsize + range_delay_size > 30*1024*1024:

range_queue.put((start, end, response))

time.sleep(10)

continue

headers['Range'] = 'bytes=%d-%d' % (start, end)

fetchserver = ''

if not response:

fetchserver = random.choice(self.fetchservers)

if self._last_app_status.get(fetchserver, 200) >= 500:

time.sleep(5)

response = self.plugin.fetch(self.handler, self.handler.command, self.url, headers, self.handler.body, timeout=self.handler.connect_timeout, fetchserver=fetchserver, **self.kwargs)

except Queue.Empty:

continue

except Exception as e:

logging.warning("RangeFetch fetch response %r in __fetchlet", e)

range_queue.put((start, end, None))

continue

if not response:

logging.warning('RangeFetch %s return %r', headers['Range'], response)

range_queue.put((start, end, None))

continue

if fetchserver:

self._last_app_status[fetchserver] = response.app_status

if response.app_status != 200:

logging.warning('Range Fetch "%s %s" %s return %s', self.handler.command, self.url, headers['Range'], response.app_status)

response.close()

range_queue.put((start, end, None))

continue

if response.getheader('Location'):

self.url = urlparse.urljoin(self.url, response.getheader('Location'))

logging.info('RangeFetch Redirect(%r)', self.url)

response.close()

range_queue.put((start, end, None))

continue

if 200 <= response.status < 300:

content_range = response.getheader('Content-Range')

if not content_range:

logging.warning('RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s', self.handler.command, self.url, content_range, response.getheaders(), start, end)

response.close()

range_queue.put((start, end, None))

continue

content_length = int(response.getheader('Content-Length', 0))

logging.info('>>>>>>>>>>>>>>> [thread %s] %s %s', threading.currentThread().ident, content_length, content_range)

while 1:

try:

if self._stopped:

response.close()

return

data = None

with gevent.Timeout(max(1, self.bufsize//8192), False):

data = response.read(self.bufsize)

if not data:

break

data_queue.put((start, data))

start += len(data)

except Exception as e:

logging.warning('RangeFetch "%s %s" %s failed: %s', self.handler.command, self.url, headers['Range'], e)

break

if start < end + 1:

logging.warning('RangeFetch "%s %s" retry %s-%s', self.handler.command, self.url, start, end)

response.close()

range_queue.put((start, end, None))

continue

logging.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.', start - 1)

else:

logging.error('RangeFetch %r return %s', self.url, response.status)

response.close()

range_queue.put((start, end, None))

continue

except StandardError as e:

logging.exception('RangeFetch._fetchlet error:%s', e)

raise

class GAEFetchPlugin(BaseFetchPlugin):

"""gae fetch plugin"""

connect_timeout = 4

max_retry = 2

def __init__(self, appids, password, path, mode, cachesock, keepalive, obfuscate, pagespeed, validate, options, maxsize):

BaseFetchPlugin.__init__(self)

self.appids = appids

self.password = password

self.path = path

self.mode = mode

self.cachesock = cachesock

self.keepalive = keepalive

self.obfuscate = obfuscate

self.pagespeed = pagespeed

self.validate = validate

self.options = options

self.maxsize = maxsize

def handle(self, handler, **kwargs):

assert handler.command != 'CONNECT'

method = handler.command

headers = dict((k.title(), v) for k, v in handler.headers.items())

body = handler.body

if handler.path[0] == '/':

url = '%s://%s%s' % (handler.scheme, handler.headers['Host'], handler.path)

elif handler.path.lower().startswith(('http://', 'https://', 'ftp://')):

url = handler.path

else:

raise ValueError('URLFETCH %r is not a valid url' % handler.path)

errors = []

response = None

for i in xrange(self.max_retry):

try:

response = self.fetch(handler, method, url, headers, body, self.connect_timeout)

if response.app_status < 500:

break

else:

if response.app_status == 503:

# appid over qouta, switch to next appid

if len(self.appids) > 1:

self.appids.append(self.appids.pop(0))

logging.info('gae over qouta, switch next appid=%r', self.appids[0])

if i < self.max_retry - 1 and len(self.appids) > 1:

self.appids.append(self.appids.pop(0))

logging.info('URLFETCH return %d, trying next appid=%r', response.app_status, self.appids[0])

response.close()

except Exception as e:

errors.append(e)

logging.info('GAE "%s %s" appid=%r %r, retry...', handler.command, handler.path, self.appids[0], e)

if len(errors) == self.max_retry:

if response and response.app_status >= 500:

status = response.app_status

headers = dict(response.getheaders())

content = response.read()

response.close()

else:

status = 502

headers = {'Content-Type': 'text/html'}

content = message_html('502 URLFetch failed', 'Local URLFetch %r failed' % handler.path, '<br>'.join(repr(x) for x in errors))

return handler.handler_plugins['mock'].handle(handler, status, headers, content)

logging.info('%s "GAE %s %s %s" %s %s', handler.address_string(), handler.command, handler.path, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))

try:

if response.status == 206:

fetchservers = ['%s://%s.appspot.com%s' % (self.mode, x, self.path) for x in self.appids]

return RangeFetch(handler, self, response, fetchservers).fetch()

handler.close_connection = not response.getheader('Content-Length')

handler.send_response(response.status)

for key, value in response.getheaders():

if key.title() == 'Transfer-Encoding':

continue

handler.send_header(key, value)

handler.end_headers()

bufsize = 8192

while True:

data = None

with gevent.Timeout(self.connect_timeout, False):

data = response.read(bufsize)

if data is None:

logging.warning('response.read(%r) %r timeout', bufsize, url)

handler.close_connection = True

break

if data:

handler.wfile.write(data)

if not data:

cache_sock = getattr(response, 'cache_sock', None)

if cache_sock:

cache_sock.close()

del response.cache_sock

response.close()

break

del data

except NetWorkIOError as e:

if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):

return

def fetch(self, handler, method, url, headers, body, timeout, **kwargs):

if isinstance(body, basestring) and body:

if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:

zbody = deflate(body)

if len(zbody) < len(body):

body = zbody

headers['Content-Encoding'] = 'deflate'

headers['Content-Length'] = str(len(body))

# GAE donot allow set `Host` header

if 'Host' in headers:

del headers['Host']

kwargs = {}

if self.password:

kwargs['password'] = self.password

if self.options:

kwargs['options'] = self.options

if self.validate:

kwargs['validate'] = self.validate

if self.maxsize:

kwargs['maxsize'] = self.maxsize

payload = '%s %s %s\r\n' % (method, url, handler.request_version)

payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in handler.skip_headers)

payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v)

# prepare GAE request

request_method = 'POST'

fetchserver_index = random.randint(0, len(self.appids)-1) if 'Range' in headers else 0

fetchserver = kwargs.get('fetchserver') or '%s://%s.appspot.com%s' % (self.mode, self.appids[fetchserver_index], self.path)

request_headers = {}

if common.GAE_OBFUSCATE:

request_method = 'GET'

fetchserver += 'ps/%d%s.gif' % (int(time.time()*1000), random.random())

request_headers['X-URLFETCH-PS1'] = base64.b64encode(deflate(payload)).strip()

if body:

request_headers['X-URLFETCH-PS2'] = base64.b64encode(deflate(body)).strip()

body = ''

if common.GAE_PAGESPEED:

fetchserver = re.sub(r'^(\w+://)', r'\g<1>1-ps.googleusercontent.com/h/', fetchserver)

else:

payload = deflate(payload)

body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body)

if 'rc4' in common.GAE_OPTIONS:

request_headers['X-URLFETCH-Options'] = 'rc4'

body = RC4Cipher(kwargs.get('password')).encrypt(body)

request_headers['Content-Length'] = str(len(body))

# post data

need_crlf = 0 if common.GAE_MODE == 'https' else 1

need_validate = common.GAE_VALIDATE

cache_key = '%s:%d' % (common.HOST_POSTFIX_MAP['.appspot.com'], 443 if common.GAE_MODE == 'https' else 80)

headfirst = bool(common.GAE_HEADFIRST)

response = handler.create_http_request(request_method, fetchserver, request_headers, body, timeout, crlf=need_crlf, validate=need_validate, cache_key=cache_key, headfirst=headfirst)

response.app_status = response.status

if response.app_status != 200:

return response

if 'rc4' in request_headers.get('X-URLFETCH-Options', ''):

response.fp = CipherFileObject(response.fp, RC4Cipher(kwargs['password']))

data = response.read(2)

if len(data) < 2:

response.status = 502

response.fp = io.BytesIO(b'connection aborted. too short leadbyte data=' + data)

response.read = response.fp.read

return response

headers_length, = struct.unpack('!h', data)

data = response.read(headers_length)

if len(data) < headers_length:

response.status = 502

response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)

response.read = response.fp.read

return response

raw_response_line, headers_data = inflate(data).split('\r\n', 1)

_, response.status, response.reason = raw_response_line.split(None, 2)

response.status = int(response.status)

response.reason = response.reason.strip()

response.msg = httplib.HTTPMessage(io.BytesIO(headers_data))

return response

class PHPFetchPlugin(BaseFetchPlugin):

"""php fetch plugin"""

connect_timeout = 4

def __init__(self, fetchservers, password, validate):

BaseFetchPlugin.__init__(self)

self.fetchservers = fetchservers

self.password = password

self.validate = validate

def handle(self, handler, **kwargs):

method = handler.command

url = handler.path

headers = dict((k.title(), v) for k, v in handler.headers.items())

body = handler.body

if body:

if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:

zbody = deflate(body)

if len(zbody) < len(body):

body = zbody

headers['Content-Encoding'] = 'deflate'

headers['Content-Length'] = str(len(body))

skip_headers = handler.skip_headers

if self.password:

kwargs['password'] = self.password

if self.validate:

kwargs['validate'] = self.validate

payload = '%s %s %s\r\n' % (method, url, handler.request_version)

payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in handler.skip_headers)

payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v)

payload = deflate(payload)

body = '%s%s%s' % ((struct.pack('!h', len(payload)), payload, body))

request_headers = {'Content-Length': len(body), 'Content-Type': 'application/octet-stream'}

fetchserver_index = 0 if 'Range' not in headers else random.randint(0, len(self.fetchservers)-1)

fetchserver = '%s?%s' % (self.fetchservers[fetchserver_index], random.random())

crlf = 0

cache_key = '%s//:%s' % urlparse.urlsplit(fetchserver)[:2]

try:

response = handler.create_http_request('POST', fetchserver, request_headers, body, self.connect_timeout, crlf=crlf, cache_key=cache_key)

except Exception as e:

logging.warning('%s "%s" failed %r', method, url, e)

return

response.app_status = response.status

need_decrypt = self.password and response.app_status == 200 and response.getheader('Content-Type', '') == 'image/gif' and response.fp

if need_decrypt:

response.fp = CipherFileObject(response.fp, XORCipher(self.password[0]))

logging.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))

handler.close_connection = bool(response.getheader('Transfer-Encoding'))

while True:

data = response.read(8192)

if not data:

break

handler.wfile.write(data)

del data

class VPSFetchPlugin(BaseFetchPlugin):

"""vps fetch plugin"""

connect_timeout = 4

def __init__(self, fetchservers, username, password):

BaseFetchPlugin.__init__(self)

self.fetchservers = fetchservers

self.username = username

self.password = password

self.fake_headers = {}

def handle(self, handler, **kwargs):

if handler.command == 'CONNECT':

return self.handle_connect(handler, **kwargs)

else:

return self.handle_method(handler, **kwargs)

def handle_connect(self, handler, **kwargs):

return

def handle_method(self, handler, **kwargs):

method = handler.command

url = handler.path

headers = dict((k.title(), v) for k, v in handler.headers.items() if k.title() not in handler.skip_headers)

x_headers = {}

if 'Host' in headers:

x_headers['Host'] = headers.pop('Host')

if 'Cookie' in headers:

x_headers['Cookie'] = headers.pop('Cookie')

headers['Host'] = 'www.%s.com' % self.username

self.fake_headers = headers.copy()

fetchserver = random.choice(self.fetchservers)

response = handler.create_http_request(handler.command, fetchserver, headers, handler.body, self.connect_timeout)

if not response:

raise socket.error(errno.ECONNRESET, 'urlfetch %r return None' % url)

#TODO

class HostsFilter(BaseProxyHandlerFilter):

"""hosts filter"""

def __init__(self, iplist_map, host_map, host_postfix_map, hostport_map, hostport_postfix_map, urlre_map):

self.iplist_map = iplist_map

self.host_map = host_map

self.host_postfix_map = host_postfix_map

self.host_postfix_endswith = tuple(host_postfix_map)

self.hostport_map = hostport_map

self.hostport_postfix_map = hostport_postfix_map

self.hostport_postfix_endswith = tuple(hostport_postfix_map)

self.urlre_map = urlre_map

def gethostbyname2(self, handler, hostname):

hostport = '%s:%d' % (hostname, handler.port)

hosts = ''

if hostname in self.host_map:

hosts = self.host_map[hostname]

elif hostname.endswith(self.host_postfix_endswith):

hosts = next(self.host_postfix_map[x] for x in self.host_postfix_map if hostname.endswith(x))

if hostport in self.hostport_map:

hosts = self.hostport_map[hostport]

elif hostport.endswith(self.hostport_postfix_endswith):

hosts = next(self.hostport_postfix_map[x] for x in self.hostport_postfix_map if hostport.endswith(x))

if handler.command != 'CONNECT' and self.urlre_map:

try:

hosts = next(self.urlre_map[x] for x in self.urlre_map if x(handler.path))

except StopIteration:

pass

if hosts not in ('', 'direct'):

return self.iplist_map.get(hosts) or hosts.split('|')

return None

def filter(self, handler):

host, port = handler.host, handler.port

hostport = handler.path if handler.command == 'CONNECT' else '%s:%d' % (host, port)

headfirst = '.google' in host

if host in self.host_map:

return 'direct', {'cache_key': '%s:%d' % (self.host_map[host], port), 'headfirst': headfirst}

elif host.endswith(self.host_postfix_endswith):

self.host_map[host] = next(self.host_postfix_map[x] for x in self.host_postfix_map if host.endswith(x))

return 'direct', {'cache_key': '%s:%d' % (self.host_map[host], port), 'headfirst': headfirst}

elif hostport in self.hostport_map:

return 'direct', {'cache_key': '%s:%d' % (self.hostport_map[hostport], port), 'headfirst': headfirst}

elif hostport.endswith(self.hostport_postfix_endswith):

self.hostport_map[hostport] = next(self.hostport_postfix_map[x] for x in self.hostport_postfix_map if hostport.endswith(x))

return 'direct', {'cache_key': '%s:%d' % (self.hostport_map[hostport], port), 'headfirst': headfirst}

if handler.command != 'CONNECT' and self.urlre_map and any(x(handler.path) for x in self.urlre_map):

return 'direct', {'headfirst': headfirst}

class GAEFetchFilter(BaseProxyHandlerFilter):

"""gae fetch filter"""

#https://github.com/AppScale/gae_sdk/blob/master/google/appengine/api/taskqueue/taskqueue.py#L241

MAX_URL_LENGTH = 2083

def filter(self, handler):

"""https://developers.google.com/appengine/docs/python/urlfetch/"""

if handler.command == 'CONNECT':

do_ssl_handshake = 440 <= handler.port <= 450 or 1024 <= handler.port <= 65535

return 'strip', {'do_ssl_handshake': do_ssl_handshake}

elif handler.command in ('GET', 'POST', 'HEAD', 'PUT', 'DELETE', 'PATCH'):

return 'gae', {}

else:

if 'php' in handler.handler_plugins:

return 'php', {}

else:

logging.warning('"%s %s" not supported by GAE, please enable PHP mode!', handler.command, handler.path)

return 'direct', {}

class WithGAEFilter(BaseProxyHandlerFilter):

"""withgae/withphp/withvps filter"""

def __init__(self, withgae_sites, withphp_sites, withvps_sites):

self.withgae_sites = set(x for x in withgae_sites if not x.startswith('.'))

self.withgae_sites_postfix = tuple(x for x in withgae_sites if x.startswith('.'))

self.withphp_sites = set(x for x in withphp_sites if not x.startswith('.'))

self.withphp_sites_postfix = tuple(x for x in withphp_sites if x.startswith('.'))

self.withvps_sites = set(x for x in withvps_sites if not x.startswith('.'))

self.withvps_sites_postfix = tuple(x for x in withvps_sites if x.startswith('.'))

def filter(self, handler):

plugin = ''

if handler.host in self.withgae_sites or handler.host.endswith(self.withgae_sites_postfix):

plugin = 'gae'

elif handler.host in self.withphp_sites or handler.host.endswith(self.withphp_sites_postfix):

plugin = 'php'

elif handler.host in self.withvps_sites or handler.host.endswith(self.withvps_sites_postfix):

plugin = 'vps'

if plugin:

if handler.command == 'CONNECT':

do_ssl_handshake = 440 <= handler.port <= 450 or 1024 <= handler.port <= 65535

return 'strip', {'do_ssl_handshake': do_ssl_handshake}

else:

return plugin, {}

class GAEProxyHandler(MultipleConnectionMixin, SimpleProxyHandler):

"""GAE Proxy Handler"""

handler_filters = [GAEFetchFilter()]

handler_plugins = {'direct': DirectFetchPlugin(),

'mock': MockFetchPlugin(),

'strip': StripPlugin(),}

hosts_filter = None

def __init__(self, *args, **kwargs):

SimpleProxyHandler.__init__(self, *args, **kwargs)

def first_run(self):

"""GAEProxyHandler setup, init domain/iplist map"""

openssl_set_session_cache_mode(self.openssl_context, 'client')

if not common.PROXY_ENABLE:

logging.info('resolve common.IPLIST_MAP names=%s to iplist', list(common.IPLIST_MAP))

common.resolve_iplist()

random.shuffle(common.GAE_APPIDS)

self.__class__.handler_plugins['gae'] = GAEFetchPlugin(common.GAE_APPIDS, common.GAE_PASSWORD, common.GAE_PATH, common.GAE_MODE, common.GAE_CACHESOCK, common.GAE_KEEPALIVE, common.GAE_OBFUSCATE, common.GAE_PAGESPEED, common.GAE_VALIDATE, common.GAE_OPTIONS, common.GAE_MAXSIZE)

try:

self.__class__.hosts_filter = next(x for x in self.__class__.handler_filters if isinstance(x, HostsFilter))

except StopIteration:

pass

def gethostbyname2(self, hostname):

iplist = self.hosts_filter.gethostbyname2(self, hostname) if self.hosts_filter else None

return iplist or MultipleConnectionMixin.gethostbyname2(self, hostname)

class ProxyGAEProxyHandler(ProxyConnectionMixin, GAEProxyHandler):

def __init__(self, *args, **kwargs):

ProxyConnectionMixin.__init__(self, common.PROXY_HOST, common.PROXY_PORT, common.PROXY_USERNAME, common.PROXY_PASSWROD)

GAEProxyHandler.__init__(self, *args, **kwargs)

def gethostbyname2(self, hostname):

for postfix in ('.appspot.com', '.googleusercontent.com'):

if hostname.endswith(postfix):

host = common.HOST_MAP.get(hostname) or common.HOST_POSTFIX_MAP.get(postfix) or 'www.google.com'

return common.IPLIST_MAP.get(host) or host.split('|')

return ProxyConnectionMixin.gethostbyname2(self, hostname)

class PHPFetchFilter(BaseProxyHandlerFilter):

"""php fetch filter"""

def filter(self, handler):

if handler.command == 'CONNECT':

return 'strip', {}

else:

return 'php', {}

class VPSFetchFilter(BaseProxyHandlerFilter):

"""vps fetch filter"""

def filter(self, handler):

return 'vps', {}

class PHPProxyHandler(MultipleConnectionMixin, SimpleProxyHandler):

"""PHP Proxy Handler"""

handler_filters = [PHPFetchFilter()]

handler_plugins = {'direct': DirectFetchPlugin(),

'mock': MockFetchPlugin(),

'strip': StripPlugin(),}

def __init__(self, *args, **kwargs):

SimpleProxyHandler.__init__(self, *args, **kwargs)

class ProxyPHPProxyHandler(ProxyConnectionMixin, PHPProxyHandler):

def __init__(self, *args, **kwargs):

ProxyConnectionMixin.__init__(self, common.PROXY_HOST, common.PROXY_PORT, common.PROXY_USERNAME, common.PROXY_PASSWROD)

PHPProxyHandler.__init__(self, *args, **kwargs)

def gethostbyname2(self, hostname):

return [hostname]

class Common(object):

"""Global Config Object"""

ENV_CONFIG_PREFIX = 'GOAGENT_'

def __init__(self):

"""load config from proxy.ini"""

ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>\S+)\s+(?P<vi>[=])\s+(?P<value>.*)$')

self.CONFIG = ConfigParser.ConfigParser()

self.CONFIG_FILENAME = os.path.splitext(os.path.abspath(__file__))[0]+'.ini'

self.DATA_PATH = os.path.join(root_path, "data", "php_proxy")

if not os.path.isdir(self.DATA_PATH):

os.mkdir(self.DATA_PATH)

self.CONFIG_USER_FILENAME = os.path.join(self.DATA_PATH, "config.ini")

if os.path.isfile(self.CONFIG_USER_FILENAME):

self.CONFIG.read([self.CONFIG_FILENAME, self.CONFIG_USER_FILENAME])

else:

self.CONFIG.read(self.CONFIG_FILENAME)

for key, value in os.environ.items():

m = re.match(r'^%s([A-Z]+)_([A-Z\_\-]+)$' % self.ENV_CONFIG_PREFIX, key)

if m:

self.CONFIG.set(m.group(1).lower(), m.group(2).lower(), value)

self.LISTEN_IP = self.CONFIG.get('listen', 'ip')

self.LISTEN_PORT = self.CONFIG.getint('listen', 'port')

self.LISTEN_USERNAME = self.CONFIG.get('listen', 'username') if self.CONFIG.has_option('listen', 'username') else ''

self.LISTEN_PASSWORD = self.CONFIG.get('listen', 'password') if self.CONFIG.has_option('listen', 'password') else ''

self.LISTEN_VISIBLE = self.CONFIG.getint('listen', 'visible')

self.LISTEN_DEBUGINFO = self.CONFIG.getint('listen', 'debuginfo')

self.GAE_ENABLE = self.CONFIG.getint('gae', 'enable')

self.GAE_APPIDS = re.findall(r'[\w\-\.]+', self.CONFIG.get('gae', 'appid').replace('.appspot.com', ''))

self.GAE_PASSWORD = self.CONFIG.get('gae', 'password').strip()

self.GAE_PATH = self.CONFIG.get('gae', 'path')

self.GAE_MODE = self.CONFIG.get('gae', 'mode')

self.GAE_IPV6 = self.CONFIG.getint('gae', 'ipv6')

self.GAE_WINDOW = self.CONFIG.getint('gae', 'window')

self.GAE_KEEPALIVE = self.CONFIG.getint('gae', 'keepalive')

self.GAE_CACHESOCK = self.CONFIG.getint('gae', 'cachesock')

self.GAE_HEADFIRST = self.CONFIG.getint('gae', 'headfirst')

self.GAE_OBFUSCATE = self.CONFIG.getint('gae', 'obfuscate')

self.GAE_VALIDATE = self.CONFIG.getint('gae', 'validate')

self.GAE_TRANSPORT = self.CONFIG.getint('gae', 'transport') if self.CONFIG.has_option('gae', 'transport') else 0

self.GAE_OPTIONS = self.CONFIG.get('gae', 'options')

self.GAE_REGIONS = set(x.upper() for x in self.CONFIG.get('gae', 'regions').split('|') if x.strip())

self.GAE_SSLVERSION = self.CONFIG.get('gae', 'sslversion')

self.GAE_PAGESPEED = self.CONFIG.getint('gae', 'pagespeed') if self.CONFIG.has_option('gae', 'pagespeed') else 0

self.GAE_MAXSIZE = self.CONFIG.getint('gae', 'maxsize')

if self.GAE_IPV6:

sock = None

try:

sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)

sock.connect(('2001:4860:4860::8888', 53))

logging.info('use ipv6 interface %s for gae', sock.getsockname()[0])

except Exception as e:

logging.info('Fail try use ipv6 %r, fallback ipv4', e)

self.GAE_IPV6 = 0

finally:

if sock:

sock.close()

if 'USERDNSDOMAIN' in os.environ and re.match(r'^\w+\.\w+$', os.environ['USERDNSDOMAIN']):

self.CONFIG.set('profile', '.' + os.environ['USERDNSDOMAIN'], '')

urlrewrite_map = collections.OrderedDict()

host_map = collections.OrderedDict()

host_postfix_map = collections.OrderedDict()

hostport_map = collections.OrderedDict()

hostport_postfix_map = collections.OrderedDict()

urlre_map = collections.OrderedDict()

withgae_sites = []

withphp_sites = []

withvps_sites = []

crlf_sites = []

nocrlf_sites = []

forcehttps_sites = []

noforcehttps_sites = []

fakehttps_sites = []

nofakehttps_sites = []

dns_servers = []

for site, rule in self.CONFIG.items('profile'):

rules = [x.strip() for x in re.split(r'[,\|]', rule) if x.strip()]

if site == 'dns':

dns_servers = rules

continue

if rule.startswith(('file://', 'http://', 'https://')) or '$1' in rule:

urlrewrite_map[site] = rule

continue

for name, sites in [('withgae', withgae_sites),

('withphp', withphp_sites),

('withvps', withvps_sites),

('crlf', crlf_sites),

('nocrlf', nocrlf_sites),

('forcehttps', forcehttps_sites),

('noforcehttps', noforcehttps_sites),

('fakehttps', fakehttps_sites),

('nofakehttps', nofakehttps_sites)]:

if name in rules:

sites.append(site)

rules.remove(name)

hostname = rules and rules[0]

if not hostname:

continue

if ':' in site and '\\' not in site:

if site.startswith('.'):

hostport_postfix_map[site] = hostname

else:

hostport_map[site] = hostname

elif '\\' in site:

urlre_map[re.compile(site).match] = hostname

else:

if site.startswith('.'):

host_postfix_map[site] = hostname

else:

host_map[site] = hostname

self.HTTP_DNS = dns_servers

self.WITHGAE_SITES = tuple(withgae_sites)

self.WITHPHP_SITES = tuple(withphp_sites)

self.WITHVPS_SITES = tuple(withvps_sites)

self.CRLF_SITES = tuple(crlf_sites)

self.NOCRLF_SITES = set(nocrlf_sites)

self.FORCEHTTPS_SITES = tuple(forcehttps_sites)

self.NOFORCEHTTPS_SITES = set(noforcehttps_sites)

self.FAKEHTTPS_SITES = tuple(fakehttps_sites)

self.NOFAKEHTTPS_SITES = set(nofakehttps_sites)

self.URLREWRITE_MAP = urlrewrite_map

self.HOSTPORT_MAP = hostport_map

self.HOSTPORT_POSTFIX_MAP = hostport_postfix_map

self.URLRE_MAP = urlre_map

self.HOST_MAP = host_map

self.HOST_POSTFIX_MAP = host_postfix_map

self.IPLIST_MAP = collections.OrderedDict((k, v.split('|') if v else []) for k, v in self.CONFIG.items('iplist'))

self.IPLIST_MAP.update((k, [k]) for k, v in self.HOST_MAP.items() if k == v)

self.IPLIST_PREDEFINED = [x for x in sum(self.IPLIST_MAP.values(), []) if re.match(r'^\d+\.\d+\.\d+\.\d+$', x) or ':' in x]

if self.GAE_IPV6 and 'google_ipv6' in self.IPLIST_MAP:

for name in self.IPLIST_MAP.keys():

if name.startswith('google') and name not in ('google_ipv6', 'google_talk'):

self.IPLIST_MAP[name] = self.IPLIST_MAP['google_ipv6']

self.PAC_ENABLE = self.CONFIG.getint('pac', 'enable')

self.PAC_IP = self.CONFIG.get('pac', 'ip')

self.PAC_PORT = self.CONFIG.getint('pac', 'port')

self.PAC_FILE = self.CONFIG.get('pac', 'file').lstrip('/')

self.PAC_GFWLIST = self.CONFIG.get('pac', 'gfwlist')

self.PAC_ADBLOCK = self.CONFIG.get('pac', 'adblock')

self.PAC_ADMODE = self.CONFIG.getint('pac', 'admode')

self.PAC_EXPIRED = self.CONFIG.getint('pac', 'expired')

self.PHP_ENABLE = self.CONFIG.getint('php', 'enable')

self.PHP_LISTEN = self.CONFIG.get('php', 'listen')

self.PHP_PASSWORD = self.CONFIG.get('php', 'password') if self.CONFIG.has_option('php', 'password') else ''

self.PHP_CRLF = self.CONFIG.getint('php', 'crlf') if self.CONFIG.has_option('php', 'crlf') else 1

self.PHP_VALIDATE = self.CONFIG.getint('php', 'validate') if self.CONFIG.has_option('php', 'validate') else 0

self.PHP_FETCHSERVERS = self.CONFIG.get('php', 'fetchserver').split('|')

self.PROXY_ENABLE = self.CONFIG.getint('proxy', 'enable')

self.PROXY_AUTODETECT = self.CONFIG.getint('proxy', 'autodetect') if self.CONFIG.has_option('proxy', 'autodetect') else 0

self.PROXY_HOST = self.CONFIG.get('proxy', 'host')

self.PROXY_PORT = self.CONFIG.get('proxy', 'port')

if self.PROXY_PORT == "":

self.PROXY_PORT = 0

else:

self.PROXY_PORT = int(self.PROXY_PORT)

self.PROXY_USERNAME = self.CONFIG.get('proxy', 'username')

self.PROXY_PASSWROD = self.CONFIG.get('proxy', 'password')

if not self.PROXY_ENABLE and self.PROXY_AUTODETECT:

system_proxy = ProxyUtil.get_system_proxy()

if system_proxy and self.LISTEN_IP not in system_proxy:

_, username, password, address = ProxyUtil.parse_proxy(system_proxy)

proxyhost, _, proxyport = address.rpartition(':')

self.PROXY_ENABLE = 1

self.PROXY_USERNAME = username

self.PROXY_PASSWROD = password

self.PROXY_HOST = proxyhost

self.PROXY_PORT = int(proxyport)

if self.PROXY_ENABLE:

self.GAE_MODE = 'https'

self.CONTROL_ENABLE = self.CONFIG.getint('control', 'enable')

self.CONTROL_IP = self.CONFIG.get('control', 'ip')

self.CONTROL_PORT = self.CONFIG.getint('control', 'port')

self.AUTORANGE_HOSTS = self.CONFIG.get('autorange', 'hosts').split('|')

self.AUTORANGE_ENDSWITH = tuple(self.CONFIG.get('autorange', 'endswith').split('|'))

self.AUTORANGE_NOENDSWITH = tuple(self.CONFIG.get('autorange', 'noendswith').split('|'))

self.AUTORANGE_MAXSIZE = self.CONFIG.getint('autorange', 'maxsize')

self.AUTORANGE_WAITSIZE = self.CONFIG.getint('autorange', 'waitsize')

self.AUTORANGE_BUFSIZE = self.CONFIG.getint('autorange', 'bufsize')

self.AUTORANGE_THREADS = self.CONFIG.getint('autorange', 'threads')

self.FETCHMAX_LOCAL = self.CONFIG.getint('fetchmax', 'local') if self.CONFIG.get('fetchmax', 'local') else 3

self.FETCHMAX_SERVER = self.CONFIG.get('fetchmax', 'server')

self.DNS_ENABLE = self.CONFIG.getint('dns', 'enable')

self.DNS_LISTEN = self.CONFIG.get('dns', 'listen')

self.DNS_SERVERS = self.HTTP_DNS or self.CONFIG.get('dns', 'servers').split('|')

self.DNS_BLACKLIST = set(self.CONFIG.get('dns', 'blacklist').split('|'))

self.DNS_TCPOVER = tuple(self.CONFIG.get('dns', 'tcpover').split('|')) if self.CONFIG.get('dns', 'tcpover').strip() else tuple()

if self.GAE_IPV6:

self.DNS_SERVERS = [x for x in self.DNS_SERVERS if ':' in x]

else:

self.DNS_SERVERS = [x for x in self.DNS_SERVERS if ':' not in x]

self.USERAGENT_ENABLE = self.CONFIG.getint('useragent', 'enable')

self.USERAGENT_STRING = self.CONFIG.get('useragent', 'string')

self.LOVE_ENABLE = self.CONFIG.getint('love', 'enable')

self.LOVE_TIP = self.CONFIG.get('love', 'tip').encode('utf8').decode('unicode-escape').split('|')

self.keep_run = True

def extend_iplist(self, iplist_name, hosts):

logging.info('extend_iplist start for hosts=%s', hosts)

new_iplist = []