|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# Copyright (C) 2011, 2012 Povilas Kanapickas <povilas@radix.lt> |
| 4 | +# |
| 5 | +# This file is part of cppreference-doc |
| 6 | +# |
| 7 | +# This program is free software: you can redistribute it and/or modify |
| 8 | +# it under the terms of the GNU General Public License as published by |
| 9 | +# the Free Software Foundation, either version 3 of the License, or |
| 10 | +# (at your option) any later version. |
| 11 | +# |
| 12 | +# This program is distributed in the hope that it will be useful, |
| 13 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | +# GNU General Public License for more details. |
| 16 | +# |
| 17 | +# You should have received a copy of the GNU General Public License |
| 18 | +# along with this program. If not, see http://www.gnu.org/licenses/. |
| 19 | + |
| 20 | +import fnmatch |
| 21 | +from lxml import etree |
| 22 | +import re |
| 23 | +import os |
| 24 | +import sys |
| 25 | +import shutil |
| 26 | +import urllib.parse |
| 27 | +from xml_utils import xml_escape, xml_unescape |
| 28 | + |
| 29 | +def rmtree_if_exists(dir): |
| 30 | + if os.path.isdir(dir): |
| 31 | + shutil.rmtree(dir) |
| 32 | + |
| 33 | +def move_dir_contents_to_dir(srcdir, dstdir): |
| 34 | + for fn in os.listdir(srcdir): |
| 35 | + shutil.move(os.path.join(srcdir, fn), |
| 36 | + os.path.join(dstdir, fn)) |
| 37 | + |
| 38 | +def rearrange_archive(root): |
| 39 | + # rearrange the archive. {root} here is output/reference |
| 40 | + |
| 41 | + # before |
| 42 | + # {root}/en.cppreference.com/w/ : html |
| 43 | + # {root}/en.cppreference.com/mwiki/ : data |
| 44 | + # {root}/en.cppreference.com/ : data |
| 45 | + # ... (other languages) |
| 46 | + # {root}/upload.cppreference.com/mwiki/ : data |
| 47 | + |
| 48 | + # after |
| 49 | + # {root}/common/ : all common data |
| 50 | + # {root}/en/ : html for en |
| 51 | + # ... (other languages) |
| 52 | + |
| 53 | + data_path = os.path.join(root, 'common') |
| 54 | + rmtree_if_exists(data_path) |
| 55 | + shutil.move(os.path.join(root, 'upload.cppreference.com/mwiki'), data_path) |
| 56 | + shutil.rmtree(os.path.join(root, 'upload.cppreference.com')) |
| 57 | + |
| 58 | + for lang in ["en"]: |
| 59 | + path = os.path.join(root, lang + ".cppreference.com/") |
| 60 | + src_html_path = path + "w/" |
| 61 | + src_data_path = path + "mwiki/" |
| 62 | + html_path = os.path.join(root, lang) |
| 63 | + |
| 64 | + if os.path.isdir(src_html_path): |
| 65 | + shutil.move(src_html_path, html_path) |
| 66 | + |
| 67 | + if os.path.isdir(src_data_path): |
| 68 | + # the skin files should be the same for all languages thus we |
| 69 | + # can merge everything |
| 70 | + move_dir_contents_to_dir(src_data_path, data_path) |
| 71 | + |
| 72 | + # also copy the custom fonts |
| 73 | + shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed60.ttf'), data_path) |
| 74 | + shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed75.ttf'), data_path) |
| 75 | + |
| 76 | + # remove what's left |
| 77 | + shutil.rmtree(path) |
| 78 | + |
| 79 | + # remove the XML source file |
| 80 | + for fn in fnmatch.filter(os.listdir(root), 'cppreference-export*.xml'): |
| 81 | + os.remove(os.path.join(root, fn)) |
| 82 | + |
| 83 | +def add_file_to_rename_map(rename_map, dir, fn, new_fn): |
| 84 | + path = os.path.join(dir, fn) |
| 85 | + if not os.path.isfile(path): |
| 86 | + print("ERROR: Not renaming '{0}' because path does not exist".format(path)) |
| 87 | + return |
| 88 | + rename_map.append((dir, fn, new_fn)) |
| 89 | + |
| 90 | +def convert_loader_name(fn): |
| 91 | + if re.search("modules=site&only=scripts", fn): |
| 92 | + return "site_scripts.js" |
| 93 | + elif re.search("modules=site&only=styles", fn): |
| 94 | + return "site_modules.css" |
| 95 | + elif re.search("modules=skins.*&only=scripts", fn): |
| 96 | + return "skin_scripts.js" |
| 97 | + elif re.search("modules=startup&only=scripts", fn): |
| 98 | + return "startup_scripts.js" |
| 99 | + elif re.search("modules=.*ext.*&only=styles", fn): |
| 100 | + return "ext.css" |
| 101 | + else: |
| 102 | + print("Loader file " + fn + " does not match any known files") |
| 103 | + sys.exit(1) |
| 104 | + |
| 105 | +def find_files_to_be_renamed(root): |
| 106 | + # Returns a rename map: array of tuples each of which contain three strings: |
| 107 | + # the directory the file resides in, the source and destination filenames. |
| 108 | + |
| 109 | + # The rename map specifies files to be renamed in order to support them on |
| 110 | + # windows filesystems which don't support certain characters in file names |
| 111 | + rename_map = [] |
| 112 | + |
| 113 | + files_rename = [] # general files to be renamed |
| 114 | + files_loader = [] # files served by load.php. These should map to |
| 115 | + # consistent and short file names because we |
| 116 | + # modify some of them later in the pipeline |
| 117 | + |
| 118 | + for dir, dirnames, filenames in os.walk(root): |
| 119 | + filenames_loader = set(fnmatch.filter(filenames, 'load.php[?]*')) |
| 120 | + # match any filenames with '?"*' characters |
| 121 | + filenames_rename = set(fnmatch.filter(filenames, '*[?"*]*')) |
| 122 | + |
| 123 | + # don't process load.php files in general rename handler |
| 124 | + filenames_rename -= filenames_loader |
| 125 | + |
| 126 | + for fn in filenames_loader: |
| 127 | + files_loader.append((dir, fn)) |
| 128 | + for fn in filenames_rename: |
| 129 | + files_rename.append((dir, fn)) |
| 130 | + |
| 131 | + for dir,orig_fn in files_rename: |
| 132 | + fn = orig_fn |
| 133 | + fn = re.sub('\?.*', '', fn) |
| 134 | + fn = re.sub('"', '_q_', fn) |
| 135 | + fn = re.sub('\*', '_star_', fn) |
| 136 | + add_file_to_rename_map(rename_map, dir, orig_fn, fn) |
| 137 | + |
| 138 | + # map loader names to more recognizable names |
| 139 | + for dir,fn in files_loader: |
| 140 | + new_fn = convert_loader_name(fn) |
| 141 | + add_file_to_rename_map(rename_map, dir, fn, new_fn) |
| 142 | + |
| 143 | + # rename filenames that conflict on case-insensitive filesystems |
| 144 | + # TODO: perform this automatically |
| 145 | + add_file_to_rename_map(rename_map, os.path.join(root, 'en/cpp/numeric/math'), 'NAN.html', 'NAN.2.html') |
| 146 | + add_file_to_rename_map(rename_map, os.path.join(root, 'en/c/numeric/math'), 'NAN.html', 'NAN.2.html') |
| 147 | + return rename_map |
| 148 | + |
| 149 | +def rename_files(rename_map): |
| 150 | + for dir, old_fn, new_fn in rename_map: |
| 151 | + src_path = os.path.join(dir, old_fn) |
| 152 | + dst_path = os.path.join(dir, new_fn) |
| 153 | + print("Renaming '{0}' to \n '{1}'".format(src_path, dst_path)) |
| 154 | + shutil.move(src_path, dst_path) |
| 155 | + |
| 156 | +def find_html_files(root): |
| 157 | + # find files that need to be preprocessed |
| 158 | + html_files = [] |
| 159 | + for dir, dirnames, filenames in os.walk(root): |
| 160 | + for filename in fnmatch.filter(filenames, '*.html'): |
| 161 | + html_files.append(os.path.join(dir, filename)) |
| 162 | + return html_files |
| 163 | + |
| 164 | +def fix_relative_link(rename_map, target, file, root): |
| 165 | + external_link_patterns = [ |
| 166 | + 'http://', |
| 167 | + 'https://', |
| 168 | + 'ftp://' |
| 169 | + ] |
| 170 | + if re.match('https?://[a-z]+\.cppreference\.com/mwiki/load\.php', target): |
| 171 | + # Absolute loader.php links need to be made relative |
| 172 | + abstarget = os.path.join(root, "common/" + convert_loader_name(target)) |
| 173 | + return os.path.relpath(abstarget, os.path.dirname(file)) |
| 174 | + else: |
| 175 | + for pattern in external_link_patterns: |
| 176 | + if pattern in target: |
| 177 | + return target |
| 178 | + |
| 179 | + target = urllib.parse.unquote(target) |
| 180 | + for dir,fn,new_fn in rename_map: |
| 181 | + target = target.replace(fn, new_fn) |
| 182 | + target = target.replace('../../upload.cppreference.com/mwiki/','../common/') |
| 183 | + target = target.replace('../mwiki/','../common/') |
| 184 | + target = re.sub('(\.php|\.css)\?.*', '\\1', target) |
| 185 | + target = urllib.parse.quote(target) |
| 186 | + target = target.replace('%23', '#') |
| 187 | + return target |
| 188 | + |
| 189 | +def has_class(el, classes_to_check): |
| 190 | + value = el.get('class') |
| 191 | + if value is None: |
| 192 | + return False |
| 193 | + classes = value.split(' ') |
| 194 | + for cl in classes_to_check: |
| 195 | + if cl in classes: |
| 196 | + return True |
| 197 | + return False |
| 198 | + |
| 199 | +def preprocess_html_file(root, fn, rename_map): |
| 200 | + |
| 201 | + parser = etree.HTMLParser() |
| 202 | + html = etree.parse(fn, parser) |
| 203 | + |
| 204 | + # remove non-printable elements |
| 205 | + for el in html.xpath('//*'): |
| 206 | + if has_class(el, ['noprint', 'editsection']): |
| 207 | + el.getparent().remove(el) |
| 208 | + if el.get('id') == 'toc': |
| 209 | + el.getparent().remove(el) |
| 210 | + |
| 211 | + # remove see also links between C and C++ documentations |
| 212 | + for el in html.xpath('//tr[@class]'): |
| 213 | + if not has_class(el, ['t-dcl-list-item']): |
| 214 | + continue |
| 215 | + |
| 216 | + child_tds = el.xpath('.//td/div[@class]') |
| 217 | + if not any(has_class(td, ['t-dcl-list-see']) for td in child_tds): |
| 218 | + continue |
| 219 | + |
| 220 | + # remove preceding separator, if any |
| 221 | + prev = el.getprevious() |
| 222 | + if prev is not None: |
| 223 | + child_tds = prev.xpath('.//td[@class') |
| 224 | + if any(has_class(td, 't-dcl-list-sep') for td in child_tds): |
| 225 | + prev.getparent().remove(prev) |
| 226 | + |
| 227 | + el.getparent().remove(el) |
| 228 | + |
| 229 | + for el in html.xpath('//h3'): |
| 230 | + if len(el.xpath(".//span[@id = 'See_also']")) == 0: |
| 231 | + continue |
| 232 | + |
| 233 | + next = el.getnext() |
| 234 | + if next is None: |
| 235 | + el.getparent().remove(el) |
| 236 | + continue |
| 237 | + |
| 238 | + if next.tag != 'table': |
| 239 | + continue |
| 240 | + |
| 241 | + if not has_class(next, 't-dcl-list-begin'): |
| 242 | + continue |
| 243 | + |
| 244 | + if len(next.xpath('.//tr')) > 0: |
| 245 | + continue |
| 246 | + |
| 247 | + el.getparent().remove(el) |
| 248 | + next.getparent().remove(next) |
| 249 | + |
| 250 | + # remove external links to unused resources |
| 251 | + for el in html.xpath('/html/head/link'): |
| 252 | + if el.get('rel') in [ 'alternate', 'search', 'edit', 'EditURI' ]: |
| 253 | + el.getparent().remove(el) |
| 254 | + |
| 255 | + # remove Google Analytics scripts |
| 256 | + for el in html.xpath('/html/body/script'): |
| 257 | + if el.get('src') is not None and 'google-analytics.com/ga.js' in el.get('src'): |
| 258 | + el.getparent().remove(el) |
| 259 | + elif el.text is not None and ('google-analytics.com/ga.js' in el.text or 'pageTracker' in el.text): |
| 260 | + el.getparent().remove(el) |
| 261 | + |
| 262 | + # apply changes to links caused by file renames |
| 263 | + for el in html.xpath('//*[@src or @href]'): |
| 264 | + if el.get('src') is not None: |
| 265 | + el.set('src', fix_relative_link(rename_map, el.get('src'), fn, root)) |
| 266 | + elif el.get('href') is not None: |
| 267 | + el.set('href', fix_relative_link(rename_map, el.get('href'), fn, root)) |
| 268 | + |
| 269 | + for err in parser.error_log: |
| 270 | + print("HTML WARN: {0}".format(err)) |
| 271 | + |
| 272 | + html.write(fn, encoding='utf-8', method='html') |
| 273 | + |
| 274 | +def preprocess_css_file(fn): |
| 275 | + |
| 276 | + f = open(fn, "r", encoding='utf-8') |
| 277 | + text = f.read() |
| 278 | + f.close() |
| 279 | + |
| 280 | + # note that query string is not used in css files |
| 281 | + |
| 282 | + text = text.replace('../DejaVuSansMonoCondensed60.ttf', 'DejaVuSansMonoCondensed60.ttf') |
| 283 | + text = text.replace('../DejaVuSansMonoCondensed75.ttf', 'DejaVuSansMonoCondensed75.ttf') |
| 284 | + |
| 285 | + # QT Help viewer doesn't understand nth-child |
| 286 | + text = text.replace('nth-child(1)', 'first-child') |
| 287 | + |
| 288 | + f = open(fn, "w", encoding='utf-8') |
| 289 | + f.write(text) |
| 290 | + f.close() |
0 commit comments