Skip to content

Commit 99a10be

Browse files
p12ticp12tic
committed
Gadgets/Stdrev: Add script to sync the source of test pages to repository
1 parent 86daee8 commit 99a10be

1 file changed

Lines changed: 173 additions & 0 deletions

File tree

‎gadgets/sync_tests_mwiki.py‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
#!/usr/bin/env python3
2+
3+
'''
4+
Copyright (C) 2016-2017 Povilas Kanapickas <povilas@radix.lt>
5+
6+
This file is part of cppreference.com
7+
8+
This program is free software: you can redistribute it and/or modify
9+
it under the terms of the GNU General Public License as published by
10+
the Free Software Foundation, either version 2 of the License, or
11+
(at your option) any later version.
12+
13+
This program is distributed in the hope that it will be useful,
14+
but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
GNU General Public License for more details.
17+
18+
You should have received a copy of the GNU General Public License
19+
along with this program. If not, see http://www.gnu.org/licenses/.
20+
'''
21+
22+
# This script depends on pywikibot framework. Install using
23+
# pip install pywikibot --pre
24+
25+
import os
26+
27+
# pywikibot loads 'user-config.py' file in current directory by default
28+
# disable this behavior. We want the same information to be supplied via
29+
# command line.
30+
os.environ['PYWIKIBOT2_NO_USER_CONFIG']='1'
31+
32+
import pywikibot
33+
import pywikibot.config2
34+
import pywikibot.pagegenerators
35+
import pywikibot.data.api
36+
37+
import argparse
38+
import itertools
39+
import shutil
40+
import sys
41+
42+
SYNC_DIRECTION_UPLOAD = 1
43+
SYNC_DIRECTION_DOWNLOAD = 2
44+
45+
def get_path_from_title(title):
46+
title = title.replace(' ', '_')
47+
48+
pathnames = title.replace(':', '/').split('/')
49+
pathnames = [ p for p in pathnames if p != '' ]
50+
51+
return '/'.join(pathnames) + '.mwiki'
52+
53+
def fix_whitespace(text):
54+
# Trims trailing whitespace on lines
55+
# Adds trailing newline if not present. MediaWiki strips it and we don't
56+
# want to fight with editors.
57+
58+
# Note that splitlines does not return empty line corresponding to the
59+
# trailing newline character.
60+
lines = text.splitlines()
61+
lines = [ l.rstrip() for l in lines ]
62+
return '\n'.join(lines) + '\n'
63+
64+
def sync_single_page(page, direction, dest_root):
65+
title = page.title()
66+
text = page.get(get_redirect=True)
67+
68+
dest_path = os.path.join(dest_root, get_path_from_title(title))
69+
70+
if direction == SYNC_DIRECTION_UPLOAD:
71+
if not os.path.exists(dest_path):
72+
return
73+
with open(dest_path, 'r') as file:
74+
new_text = file.read()
75+
if fix_whitespace(text) != fix_whitespace(new_text):
76+
page.put(new_text, 'sync with git')
77+
print('Uploaded {0}'.format(title))
78+
79+
elif direction == SYNC_DIRECTION_DOWNLOAD:
80+
dest_dir = os.path.dirname(dest_path)
81+
if not os.path.exists(dest_dir):
82+
os.makedirs(dest_dir)
83+
84+
with open(dest_path, 'w') as file:
85+
file.write(fix_whitespace(text))
86+
print('Downloaded {0}'.format(dest_path))
87+
88+
def remove_no_longer_existing_pages(pages, dest_root):
89+
paths = []
90+
for dir, dirnames, filenames in os.walk(dest_root):
91+
for filename in filenames:
92+
rel_path = os.path.join(os.path.relpath(dir, dest_root), filename)
93+
if rel_path.startswith('./'):
94+
rel_path = rel_path[2:]
95+
paths.append(rel_path)
96+
97+
paths = set(paths)
98+
99+
existing_paths = set([get_path_from_title(page.title()) for page in pages])
100+
deleted_paths = paths - existing_paths
101+
102+
for path in deleted_paths:
103+
os.remove(os.path.join(dest_root, path))
104+
105+
def perform_sync(url, direction, dest_root, user, password):
106+
107+
if direction == SYNC_DIRECTION_DOWNLOAD:
108+
if os.path.exists(dest_root):
109+
shutil.rmtree(dest_root)
110+
os.makedirs(dest_root)
111+
112+
# Supply information to config that would otherwise be defined in
113+
# user-config.py
114+
pywikibot.config2.family = 'cppreference'
115+
pywikibot.config2.mylang = 'en'
116+
pywikibot.config2.family_files['cppreference'] = url
117+
pywikibot.config2.step = 100
118+
pywikibot.config2.put_throttle = 0
119+
120+
site = pywikibot.Site(user=user, fam='cppreference')
121+
122+
# pywikibot.login.LoginManager seems to be not fully implemented and broken
123+
# Comments in the source suggest that data.api module contains full
124+
# implementation. Use it instead.
125+
login_manager = pywikibot.data.api.LoginManager(password=password,
126+
site=site, user=user)
127+
login_manager.login()
128+
129+
pages = itertools.chain(
130+
pywikibot.pagegenerators.AllpagesPageGenerator(namespace=0, site=site),
131+
pywikibot.pagegenerators.AllpagesPageGenerator(namespace=10, site=site)
132+
)
133+
pages = pywikibot.pagegenerators.PreloadingGenerator(pages, groupsize=100)
134+
135+
pages = list(pages)
136+
for page in pages:
137+
sync_single_page(page, direction, dest_root)
138+
139+
if direction == SYNC_DIRECTION_DOWNLOAD:
140+
remove_no_longer_existing_pages(pages, dest_root)
141+
142+
def main():
143+
parser = argparse.ArgumentParser(prog='sync_mediawiki')
144+
parser.add_argument('url', type=str,
145+
help='URL to root of a MediaWiki instance')
146+
parser.add_argument('direction', type=str,
147+
help='"upload" or "download"')
148+
parser.add_argument('destination_root', type=str,
149+
help='Destination directory to place results to')
150+
parser.add_argument('user', type=str,
151+
help='Username to perform bot operations under')
152+
parser.add_argument('password', type=str,
153+
help='User password to authenticate with')
154+
args = parser.parse_args()
155+
156+
direction = None
157+
if args.direction == 'upload':
158+
direction = SYNC_DIRECTION_UPLOAD
159+
elif args.direction == 'download':
160+
direction = SYNC_DIRECTION_DOWNLOAD
161+
else:
162+
print('Incorrect direction option. Expected "upload" or "download"')
163+
sys.exit(1)
164+
165+
if args.destination_root == '' or args.destination_root == '.':
166+
print("The output directory can not be the current directory")
167+
sys.exit(1)
168+
169+
perform_sync(args.url, direction, args.destination_root,
170+
args.user, args.password)
171+
172+
if __name__ == '__main__':
173+
main()

0 commit comments

Comments
 (0)