#! /usr/bin/env python

"""

Utility script to sync templates, macros, css and js files between

openlibrary website and repository.

USAGE:

./scripts/sync [push|pull|diff] [-s|--server server] paths

Pull all templates from openlibrary.org:

./scripts/sync pull --server http://openlibrary.org templates/*

By default all the templates and macros are saved to

openlibrary/plugins/oltemplates and css and js files are saved to

static. When the server is upstream.openlibrary.org, the templates

and macros are saved to openlibrary/plugins/upstream and css and

js files are saved to static/upstream.

Push all local templates and macros to dev.openlibrary.org:

./scripts/sync push --server http://dev.openlibrary.org templates/* macros/*

For pushing items to a server, the username and password on that server must be

specified in ~/.olrc. The .olrc file must be in the following format.

[openlibrary.org]

username=AnandBot

password=secret

[0.0.0.0:8080]

username=anand

password=anand123

"""

import _init_path

import sys

import os

import glob

import shutil

import urllib, urllib2

import simplejson

from optparse import OptionParser

from ConfigParser import ConfigParser

from openlibrary import api as olapi

class LocalSource:

def __init__(self, root, static):

self.root = root

self.static = static

self.type_root = "openlibrary/plugins/openlibrary"

def get_path(self, path):

"""Find the path in the filesystem for the specified resource path.

>>> s = LocalSource('openlibrary/plugins/oltemplates', 'static')

>>> s.get_path('templates/site.tmpl')

'openlibrary/plugins/oltemplates/templates/site.html'

>>> s.get_path('macros/RecentChanges')

'openlibrary/plugins/oltemplates/macros/RecentChanges.html'

>>> s.get_path('css/master.css')

'static/css/master.css'

"""

if path.startswith('type/'):

path = path.replace('type/', 'types/')

if path.endswith('*'):

return self.type_root + '/' + path

else:

return self.type_root + '/' + path + '.type'

elif path.startswith('templates/'):

return self.root + '/' + path.replace('.tmpl', '.html')

elif path.startswith('macros/'):

newpath = self.root + '/' + path

if not newpath.endswith('*'):

newpath += '.html'

return newpath

elif path.startswith('css/') or path.startswith('js/'):

return self.static + '/' + path

else:

return None

def expand(self, paths):

for p in paths:

if p is None:

continue

if '*' in p:

for dirpath, dirnames, filenames in os.walk(p[:-1]):

for f in filenames:

yield os.path.join(dirpath, f)

else:

yield p

def get_name(self, path):

"""Reverse of get_path.

>>> s = LocalSource('openlibrary/plugins/oltemplates', 'static')

>>> s.get_name('openlibrary/plugins/oltemplates/templates/site.html')

'templates/site.tmpl'

>>> s.get_name('openlibrary/plugins/oltemplates/macros/RecentChanges')

'macros/RecentChanges'

>>> s.get_name('static/css/master.css')

'css/master.css'

>>> s.get_name('openlibrary/plugins/openlibrary/types/page.type')

'type/page'

"""

if path.startswith(self.root):

path = path[len(self.root)+1:]

elif path.startswith(self.type_root):

path = path[len(self.type_root)+1:]

elif path.startswith(self.static):

path = path[len(self.static)+1:]

if path.endswith('.type'):

path = 'type/' + os.path.basename(path.replace('.type', ''))

elif path.startswith('templates/'):

path = path.replace('.html', '.tmpl')

elif path.startswith('macros/'):

path = path.replace('.html', '')

return path

def read(self, paths):

for p in self.expand(self.get_path(p) for p in paths):

yield self.get_name(p), open(p).read()

def write(self, data):

for name, text in data:

path = self.get_path(name)

if text is None:

if os.path.exists(path) and os.path.isfile(path):

print 'deleting', path

os.remove(path)

else:

dir = os.path.dirname(path)

if not os.path.exists(dir):

os.makedirs(dir)

print 'writing', path

f = open(path, 'w')

f.write(text)

f.close()

class RemoteSource:

def __init__(self, server):

self.server = server

if self.server.endswith('/'):

self.server = server[:-1]

def find(self, prefix):

q = {'key~': prefix, 'limit': 1000}

# until all properties and backreferences are deleted on production server

if prefix == '/type':

q['type'] = '/type/type'

paths = urllib.urlopen(self.server + '/query.json?' + urllib.urlencode(q)).read()

return [x['key'] for x in simplejson.loads(paths)]

def expand(self, paths):

for p in paths:

p = '/' + p

if p.endswith('*'):

for x in self.find(p):

yield x

else:

yield p

def to_text(self, thing):

type = thing['type']['key']

def get(d, key):

if key in d:

value = d[key]

if isinstance(value, dict) and 'value' in value:

value = value['value']

return value.replace('\r\n', '\n').replace('\r', '\n')

else:

return ''

#@@ why is this?

if thing.get('latest_revision') == 1:

return None

if type == '/type/type':

for k in ['id', 'latest_revision', 'revision', 'last_modified', 'created', 'permission', 'child_permission']:

thing.pop(k, None)

# no need to sync primitive types

if thing.get('kind') == 'primitive':

return None

from infogami.infobase.common import prepr

return prepr(thing)

if type == '/type/template':

return get(thing, 'body')

elif type == '/type/macro':

return get(thing, 'macro')

elif type == '/type/rawtext':

return get(thing, 'body')

else:

return None

def read(self, paths):

paths = list(self.expand(paths))

d = simplejson.dumps({'key': paths, '*': None, 'limit': 1000})

url = self.server + '/query.json?' + urllib.urlencode(dict(query=d))

data = urllib.urlopen(url).read()

for x in simplejson.loads(data):

yield x['key'][1:], self.to_text(x)

def get_ol(self):

config = ConfigParser()

config.read(os.path.expanduser('~/.olrc'))

ol = olapi.OpenLibrary(self.server)

ol.autologin()

return ol

def write(self, data):

def process(name, text):

key = '/' + name

if text is None:

return {'key': key, 'type': {'key': '/type/delete'}}

if key.startswith('/type/'):

return eval(text)

if key.startswith('/templates/'):

return {'key': key, 'body': {'value': text, 'type': '/type/text'}, 'type': {'key': '/type/template'}}

elif key.startswith('/macros/'):

return {'key': key, 'macro': {'value': text, 'type': '/type/text'}, 'type': {'key': '/type/macro'}}

elif key.startswith('/js/'):

return {'key': key, 'body': {'value': text, 'type': '/type/text'}, 'type': {'key': '/type/rawtext'}, 'content_type': 'text/javascript'}

elif key.startswith('/css/'):

return {'key': key, 'body': {'value': text, 'type': '/type/text'}, 'type': {'key': '/type/rawtext'}, 'content_type': 'text/css'}

else:

return None

query = [process(name, d) for name, d in data]

ol = self.get_ol()

print ol.save_many(query)

class Server:

def __init__(self, server, upstream=False):

if upstream:

self.local = LocalSource('openlibrary/plugins/upstream', 'static/upstream')

else:

self.local = LocalSource('openlibrary/plugins/oltemplates', 'static')

self.remote = RemoteSource(server)

def run(self, cmd, paths):

return getattr(self, cmd)(paths)

def pull(self, paths):

self.local.write(self.remote.read(paths))

def push(self, paths):

self.remote.write(self.local.read(paths))

def delete(self, paths):

self.remote.write([(name, None) for name, value in self.remote.read(paths)])

def diff(self, paths):

import tempfile

root = tempfile.mkdtemp('-remote')

local = LocalSource(root, root)

local.write(self.remote.read(paths))

for path in paths:

if path.endswith('*'):

path = path[:-1]

os.system('diff --strip-trailing-cr -u %s %s' % (local.get_path(path), self.local.get_path(path)))

shutil.rmtree(root)

def parse_args(args):

"""Parses the args and returns command, server and paths.

>>> parse_args('./scripts/server pull templates/* macros/*'.split())

('pull', 'http://openlibrary.org', ['templates/*', 'macros/*'])

>>> parse_args('./scripts/server pull --server http://dev.openlibrary.org templates/* macros/*'.split())

('pull', 'http://dev.openlibrary.org', ['templates/*', 'macros/*'])

>>> parse_args('./scripts/server push --foo http://dev.openlibrary.org templates/* macros/*'.split())

Traceback (most recent call last):

...

SystemExit: 2

>>> parse_args('./scripts/server badcommand macros/*'.split())

Traceback (most recent call last):

...

SystemExit: 2

"""

parser = OptionParser("./scripts/sync [push|pull|diff] [--server server] paths")

parser.add_option("-s", "--server", dest="server", default='http://openlibrary.org', help="OL server")

parser.add_option("--upstream", dest="upstream", action='store_true', default=False, help="pass this option to use templates from upstream plugin.")

cmd = args[1]

sys.argv = args[:1] + args[2:]

options, args = parser.parse_args()

server = options.server

if cmd not in ['push', 'pull', 'diff', 'delete', 'test']:

parser.error('Unknown command: %s' % repr(cmd))

if 'upstream' in options.server:

options.upstream = True

return cmd, options, args

def main(args):

if '-h' in args or '--help' in args:

print __doc__

else:

cmd, options, paths = parse_args(args)

if cmd == "test":

import doctest

doctest.testmod()

else:

server = Server(options.server, options.upstream)

return server.run(cmd, paths)

if __name__ == "__main__":

main(sys.argv)