"""

eGo PreProcessing (eGoPP)

This script opens an oedb database connection and executes different parts of eGo.

Reads python and SQL scripts and gives logging infos during the execution.

Also see corresponding BPML diagram.

This file is part of project "open_eGo DataProcessing" (https://github.com/openego/data_processing/).

It's copyrighted by the contributors recorded in the version control history:

openego/data_processing/preprocessing/eGo_PreProcessing.py

SPDX-License-Identifier: AGPL-3.0-or-later

"""

__copyright__ = "Reiner Lemoine Institut"

__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"

__url__ "https://www.gnu.org/licenses/agpl-3.0.en.html"

__author__ = "gplssm, Ludee"

import pandas as pd

import logging

import time

import os

import codecs

from tools import io

def preprocessing():

# Configure logging

logger = logging.getLogger('EEEE')

logger.setLevel(logging.INFO)

ch = logging.StreamHandler()

ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s %(message)s',

datefmt='%Y-%m-%d %I:%M:%S')

ch.setFormatter(formatter)

logger.addHandler(ch)

# get current time and inform about start

total_time = time.time()

logger.info('ego preprocessing started...')

# list of sql- and python-snippets that process the data in correct order

snippet_dir = os.path.abspath(

os.path.join(os.path.dirname(__file__),

'preprocessing'))

script_dir = os.path.abspath(

os.path.join(os.path.dirname(__file__),

'python_scripts'))

snippets = [

'ego_pre_voltage_level.sql',

'ego_pre_slp_parameters.sql'

]

# get database connection

conn = io.oedb_session(section='oedb')

# iterate over list of sql- and python-snippets and execute them

for snippet in snippets:

# timing and logging

snippet_time = time.time()

logger.info("Execute '{}' ...".format(snippet))

if os.path.splitext(snippet)[1] == '.sql':

snippet_str = open(os.path.join(snippet_dir, snippet)).read()

# execute desired sql snippet

conn.execution_options(autocommit=True).execute(snippet_str)

elif os.path.splitext(snippet)[1] == '.py':

filename = os.path.join(script_dir, snippet)

script_str = open(filename, "rb").read()

# execute desired sql snippet

exec(compile(script_str, filename, 'exec'))

else:

raise NameError('{} is neither a python nor a sql script (at least it '

'has not the right extension). Please add an extension '

'to the script name (.py or .sql)'.format(snippet))

# inform the user

logger.info('...successfully done in {:.2f} seconds.'.format(

time.time() - snippet_time))

# close database connection

conn.close()

logger.info('eGo PreProcessing script successfully executed in {:.2f} seconds'.format(

time.time() - total_time))

if __name__ == '__main__':

preprocessing()