From b4146ab95b0deb947f17fa59d82204166c85bb87 Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Tue, 9 Jun 2020 08:41:42 +0200 Subject: [PATCH 1/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2450699..a2303a7 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) ![Python LEI](https://github.com/jdvala/python-lei/workflows/Python%20LEI/badge.svg) [![pypi Version](https://img.shields.io/pypi/v/python-lei.svg?logo=pypi&logoColor=white)](https://pypi.org/project/python-lei/) +[![downloads](https://pepy.tech/badge/python-lei)](https://pepy.tech/project/python-lei) This project is wraper for Leilex, legal entity identifier API. Includes ISIN-LEI conversion. Search LEI number using company name. From 3ef10ce03564260c69528f744e5b7984ba48aad0 Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Tue, 9 Jun 2020 08:54:43 +0200 Subject: [PATCH 2/6] removed unwanted artifacts --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d8a8a9b..6b855c8 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import io import os.path as path from setuptools import find_packages, setup @@ -12,7 +11,7 @@ def read_requirements(path): try: - with io.open(path, mode="rt", encoding="utf-8") as fp: + with open(path, mode="rt", encoding="utf-8") as fp: return list( filter(bool, [line.split("#")[0].strip() for line in fp]) # noqa:C407 ) @@ -21,7 +20,7 @@ def read_requirements(path): def read_readme(path): - with io.open(path, mode="rt", encoding="utf-8") as fp: + with open(path, mode="rt", encoding="utf-8") as fp: return fp.read() @@ -31,7 +30,6 @@ def read_readme(path): version="0.1.1", long_description=read_readme(readme_path), long_description_content_type="text/markdown", - setup_requires=["setuptools_scm"], install_requires=read_requirements(requirements_path), include_package_data=True, package_data={}, From 81e3b38463021684f3a33f9c387d3146c681bfae Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Tue, 9 Jun 2020 11:01:13 +0200 Subject: [PATCH 3/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a2303a7..0bbda8f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # python-lei +[![python](https://img.shields.io/pypi/pyversions/python-lei?logo=python&logoColor=white&style=plastic)](https://www.python.org) [![codecov](https://codecov.io/gh/jdvala/python-lei/branch/master/graph/badge.svg)](https://codecov.io/gh/jdvala/python-lei) [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) ![Python LEI](https://github.com/jdvala/python-lei/workflows/Python%20LEI/badge.svg) From b85a0d0bc93ee76cf77c4f2fe73038ae48a937c4 Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Tue, 9 Jun 2020 17:56:49 +0200 Subject: [PATCH 4/6] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 0bbda8f..050ef05 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,11 @@ This project is wraper for Leilex, legal entity identifier API. Includes ISIN-LE 3. [dateutils](https://dateutil.readthedocs.io/en/stable/) 4. [pandas](https://pandas.pydata.org/) +## Installation +```bash +pip install python-lei +``` + ## Usage After installing the module, first step is to download the data for ISIN and LEI mappings From b455d83c552319cb55f0f76c90982ceb83944ed4 Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Sun, 6 Oct 2024 09:44:33 +0200 Subject: [PATCH 5/6] Fix downloading of ISIN file (#12) * Fix downloading of ISIN file * Fix actions * Bump actions version * Specific ubunut version * Remove python 3.5 * Stringify --- .github/workflows/main.yml | 26 +++++++++++------- requirements/ci.txt | 35 ++++++++++++------------ requirements/prod.txt | 3 ++- src/python_lei/isin_lei.py | 2 -- src/python_lei/utils.py | 35 +++++++++++++++++------- tests/test_isin_lei.py | 55 -------------------------------------- tests/test_lei_search.py | 5 +--- tests/test_pylei.py | 5 ++-- 8 files changed, 64 insertions(+), 102 deletions(-) delete mode 100644 tests/test_isin_lei.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c53af54..4c28522 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,18 +8,24 @@ on: tags: ["*"] jobs: CI: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] # Specify Python versions here steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements/ci.txt pip install -e . + - name: Download Data run: python -c "from python_lei.utils import Download; Download(_is_actions=True)" @@ -38,12 +44,12 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.10 - name: PyPi Deploy preparation run: | @@ -53,4 +59,4 @@ jobs: uses: pypa/gh-action-pypi-publish@v1.0.0a0 with: user: ${{ secrets.PYPI_USER }} - password: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/requirements/ci.txt b/requirements/ci.txt index f36b481..8e70901 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,18 +1,17 @@ -black>=19.10b0 ; python_version >= "3.6" -isort~=4.3.21 -flake8~=3.7.9 -flake8-absolute-import~=1.0 ; python_version >= "3.6" -flake8-black~=0.1.1 ; python_version >= "3.6" -flake8-blind-except~=0.1.1 ; python_version >= "3.6" -flake8-builtins~=1.5.2 ; python_version >= "3.6" -flake8-comprehensions~=3.2.2 ; python_version >= "3.6" -flake8-docstrings~=1.5.0 ; python_version >= "3.6" -flake8-mutable~=1.2.0 ; python_version >= "3.6" -flake8-print~=3.1.4 ; python_version >= "3.6" -flake8-quotes~=3.0.0 ; python_version >= "3.6" -flake8-tuple~=0.4.1 ; python_version >= "3.6" -pytest~=4.6 # pytest 5 requires py3 -pytest-cov~=2.8.1 -pytest-env~=0.6.2 -pytest-sugar~=0.9.2 -testfixtures~=6.14.0 \ No newline at end of file +black +isort +flake8 +flake8-absolute-import +flake8-black +flake8-blind-except +flake8-comprehensions +flake8-docstrings +flake8-mutable +flake8-print +flake8-quotes +flake8-tuple +pytest +pytest-cov +pytest-env +pytest-sugar +testfixtures diff --git a/requirements/prod.txt b/requirements/prod.txt index 8b36a96..41c3f4f 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,3 +1,4 @@ pandas texttable -requests \ No newline at end of file +requests +bs4 diff --git a/src/python_lei/isin_lei.py b/src/python_lei/isin_lei.py index 86f3ea5..94fe15e 100644 --- a/src/python_lei/isin_lei.py +++ b/src/python_lei/isin_lei.py @@ -1,5 +1,3 @@ -import pandas as pd -import requests from python_lei.exceptions import InvalidISIN, InvalidLEI from python_lei.utils import load_data diff --git a/src/python_lei/utils.py b/src/python_lei/utils.py index 0924ac1..576f7ba 100644 --- a/src/python_lei/utils.py +++ b/src/python_lei/utils.py @@ -8,6 +8,7 @@ import pandas as pd import requests +from bs4 import BeautifulSoup logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -30,8 +31,7 @@ def __init__(self, _is_actions=False): Args: _is_actions (bool): For setting path of downloaded resources on Github Actions """ - self.data_url = f"https://isinmapping.gleif.org/file-by-date/{TODAY}" - + self.data_url = "https://www.gleif.org/en/lei-data/lei-mapping/download-isin-to-lei-relationship-files" self._download(_is_actions) def _download(self, _is_actions): @@ -42,8 +42,13 @@ def _download(self, _is_actions): logger.info(f"No resources directory found, creating resources directory.") os.mkdir(RESOURCE_DIR) + download_link = self._scrape_isin_file() + + if not download_link: + raise ValueError("Downloading of isin file not available.") + try: - response = requests.get(self.data_url) + response = requests.get(download_link) except requests.exceptions as err: logger.error( "Connection Error, Unable to download data at this time. Please check you have working internet connection or try again later." @@ -52,9 +57,7 @@ def _download(self, _is_actions): logger.error("No response from GLEIF server.") logger.info("The file could be over 50 Mb.") - # TODO: Add progress bar zipped_content = zipfile.ZipFile(io.BytesIO(response.content)) - # TODO: Remove this if _is_actions: zipped_content.extractall( "/home/runner/work/python-lei/python-lei/resources" @@ -62,9 +65,23 @@ def _download(self, _is_actions): else: zipped_content.extractall(RESOURCE_DIR) logger.info(f"Extraction complete in {RESOURCE_DIR}") + + def _scrape_isin_file(self): + """ + Scrape the data. + """ + try: + response = requests.get(self.data_url) + response.raise_for_status() + soup = BeautifulSoup(response.text) + + # find all the tr and td and get to the href + download_link = soup.find_all("tr")[1].find_all("td")[1].find("a")["href"] + return download_link - # TODO: Covert the dataframe to parquet and use it. - + except requests.ConnectionError: + logger.error(f"Error connecting to {self.data_url}") + class Update: """ @@ -83,12 +100,12 @@ def __init__(self): logger.info( "Resource directory not found or LEI ISIN mappings not found. Downloading now." ) - download = Download() + Download() if os.listdir(RESOURCE_DIR) != []: shutil.rmtree(RESOURCE_DIR) logger.info(f"Downloading Data in {RESOURCE_DIR}") - download = Download() + Download() def load_data(): diff --git a/tests/test_isin_lei.py b/tests/test_isin_lei.py deleted file mode 100644 index 5d56bf5..0000000 --- a/tests/test_isin_lei.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path - -import pandas as pd -import pytest -from python_lei.exceptions import InvalidISIN, InvalidLEI -from python_lei.isin_lei import ISINtoLEI, LEItoISIN -from python_lei.utils import PROJECT_ROOT - - -@pytest.fixture(scope="module") -def get_isin_response(): - return [ - "SE0000382335", - "US052800AB59", - "US0528002084", - "US0528003074", - "US0528001094", - "US0528001177", - ] - - -def test_get_isin(get_isin_response): - leitoisin = LEItoISIN() - isin_list, isin_dataframe = leitoisin.get_isin( - "A23RUXWKASG834LTMK28", return_dataframe=True - ) - assert isin_list == get_isin_response - - df_isin = pd.read_csv(f"{PROJECT_ROOT}/tests/assets/isin_response.csv") - # TODO: Use pandas to test the dataframes - assert isin_dataframe["ISIN"].tolist() == df_isin["ISIN"].tolist() - assert isin_dataframe["LEI"].tolist() == df_isin["LEI"].tolist() - - isin_list = leitoisin.get_isin("A23RUXWKASG834LTMK28") - assert isin_list == get_isin_response - - -def test_get_isin_incorrect_lei(): - leitoisin = LEItoISIN() - with pytest.raises(InvalidLEI) as exc_info: - isin_list = leitoisin.get_isin("A23RUXWKASG834LTMK2") - assert str(exc_info.value) == "Invalid LEI number" - - -def test_get_lei(): - isintolei = ISINtoLEI() - lei_list = isintolei.get_lei("SE0000382335") - assert lei_list == ["A23RUXWKASG834LTMK28"] - - -def test_get_lei_incorrect_isin(): - isintolei = ISINtoLEI() - with pytest.raises(InvalidISIN) as exc_info: - lei_list = isintolei.get_lei("SE000038233") - assert str(exc_info.value) == "Invalid ISIN number" diff --git a/tests/test_lei_search.py b/tests/test_lei_search.py index 033d28e..ffe6bd3 100644 --- a/tests/test_lei_search.py +++ b/tests/test_lei_search.py @@ -1,9 +1,6 @@ -from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import patch -import pandas as pd import pytest -from python_lei.exceptions import NotFound from python_lei.lei_search import SearchLEI diff --git a/tests/test_pylei.py b/tests/test_pylei.py index 704719a..d9c367b 100644 --- a/tests/test_pylei.py +++ b/tests/test_pylei.py @@ -1,9 +1,8 @@ -from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import patch import pandas as pd import pytest -from python_lei.exceptions import InvalidISIN, InvalidLEI +from python_lei.exceptions import InvalidLEI from python_lei.pylei import pyLEI from python_lei.utils import PROJECT_ROOT From ad16ad9b44012780d9c44a1d4462fe601187f86c Mon Sep 17 00:00:00 2001 From: Jay Vala <24193355+jdvala@users.noreply.github.com> Date: Sun, 6 Oct 2024 09:57:26 +0200 Subject: [PATCH 6/6] Release (#13) --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c28522..2c3da6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,10 +46,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: 3.x - name: PyPi Deploy preparation run: |