From 8e38b2c82f63bc4147e283d01594a847aad13e53 Mon Sep 17 00:00:00 2001 From: Will Drevo Date: Wed, 15 Jul 2020 14:42:11 -0700 Subject: [PATCH 1/9] Exercise 2: Fixed HTML parsing error (#180) --- exercises/exercise02-Task_Dependencies.ipynb | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/exercises/exercise02-Task_Dependencies.ipynb b/exercises/exercise02-Task_Dependencies.ipynb index dee3270..08ec8dc 100644 --- a/exercises/exercise02-Task_Dependencies.ipynb +++ b/exercises/exercise02-Task_Dependencies.ipynb @@ -261,17 +261,17 @@ " url = 'https://github.com/{}/commits/master'.format(repo)\n", " response = requests.get(url)\n", " soup = BeautifulSoup(response.text, 'lxml')\n", - " df = pd.DataFrame(columns=['title', 'link'])\n", - " for g in soup.find_all(class_='commit-title'):\n", - " entry = {}\n", - " title = g.find_all(class_='message')[0]['aria-label']\n", - " entry['title'] = title\n", - " links = g.find_all(class_='issue-link')\n", - " if len(links) >= 1:\n", - " link = links[0]['data-url']\n", - " entry['link'] = link\n", - " df = df.append(pd.DataFrame(entry, index=[0]), sort=False)\n", - " \n", + " results = []\n", + " for commit_elt in soup.find_all('li', class_='commit'):\n", + " title = commit_elt.find_all('a', class_='message')[0].attrs.get('aria-label').split('\\n')[0]\n", + " link_elts = commit_elt.find_all('a', class_='issue-link')\n", + " link = None\n", + " for le in link_elts:\n", + " if 'issue' in le.attrs['href'].lower():\n", + " link = le.attrs['href']\n", + " results.append(dict(title=title, link=link))\n", + "\n", + " df = pd.DataFrame(results)\n", " df['repository'] = repo\n", " return df" ] @@ -352,4 +352,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file From 08277b62635ba8dbc469cd1c2efd7dd579ee7286 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Tue, 21 Jul 2020 11:00:26 -0700 Subject: [PATCH 2/9] Add requirements.txt and requirements.in to the repository for manual install (#181) Based on README, updates the README as well --- README.rst | 9 ++++- requirements.in | 8 ++++ requirements.txt | 97 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 requirements.in create mode 100644 requirements.txt diff --git a/README.rst b/README.rst index 49a7d5c..58d1b39 100644 --- a/README.rst +++ b/README.rst @@ -90,7 +90,14 @@ Local Setup git clone https://github.com/ray-project/tutorial.git -5. Install the following additional dependencies. +5. Install the additional dependencies. + + Either install them from the given requirements.txt + + .. code-block:: bash + pip install -r requirements.txt + + Or install them manually .. code-block:: bash diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..7bef521 --- /dev/null +++ b/requirements.in @@ -0,0 +1,8 @@ +modin +tensorflow +gym +scipy +opencv-python +bokeh +ipywidgets==6.0.0 +keras diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0413797 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,97 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +absl-py==0.9.0 # via tensorboard, tensorflow +astunparse==1.6.3 # via tensorflow +attrs==19.3.0 # via jsonschema +backcall==0.2.0 # via ipython +bleach==3.1.5 # via nbconvert +bokeh==2.1.1 # via -r requirements.in +cachetools==4.1.1 # via google-auth +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +cloudpickle==1.3.0 # via gym +decorator==4.4.2 # via ipython, traitlets +defusedxml==0.6.0 # via nbconvert +entrypoints==0.3 # via nbconvert +future==0.18.2 # via pyglet +gast==0.3.3 # via tensorflow +google-auth-oauthlib==0.4.1 # via tensorboard +google-auth==1.19.2 # via google-auth-oauthlib, tensorboard +google-pasta==0.2.0 # via tensorflow +grpcio==1.30.0 # via tensorboard, tensorflow +gym==0.17.2 # via -r requirements.in +h5py==2.10.0 # via keras, tensorflow +idna==2.10 # via requests +ipykernel==5.3.3 # via ipywidgets, notebook +ipython-genutils==0.2.0 # via nbformat, notebook, traitlets +ipython==7.16.1 # via ipykernel, ipywidgets +ipywidgets==6.0.0 # via -r requirements.in +jedi==0.17.2 # via ipython +jinja2==2.11.2 # via bokeh, nbconvert, notebook +jsonschema==3.2.0 # via nbformat +jupyter-client==6.1.6 # via ipykernel, notebook +jupyter-core==4.6.3 # via jupyter-client, nbconvert, nbformat, notebook +keras-preprocessing==1.1.2 # via tensorflow +keras==2.4.3 # via -r requirements.in +markdown==3.2.2 # via tensorboard +markupsafe==1.1.1 # via jinja2 +mistune==0.8.4 # via nbconvert +modin==0.7.4 # via -r requirements.in +nbconvert==5.6.1 # via notebook +nbformat==5.0.7 # via ipywidgets, nbconvert, notebook +notebook==6.0.3 # via widgetsnbextension +numpy==1.19.0 # via bokeh, gym, h5py, keras, keras-preprocessing, opencv-python, opt-einsum, pandas, scipy, tensorboard, tensorflow +oauthlib==3.1.0 # via requests-oauthlib +opencv-python==4.3.0.36 # via -r requirements.in +opt-einsum==3.3.0 # via tensorflow +packaging==20.4 # via bleach, bokeh, modin +pandas==1.0.5 # via modin +pandocfilters==1.4.2 # via nbconvert +parso==0.7.0 # via jedi +pexpect==4.8.0 # via ipython +pickleshare==0.7.5 # via ipython +pillow==7.2.0 # via bokeh +prometheus-client==0.8.0 # via notebook +prompt-toolkit==3.0.5 # via ipython +protobuf==3.12.2 # via tensorboard, tensorflow +ptyprocess==0.6.0 # via pexpect, terminado +pyasn1-modules==0.2.8 # via google-auth +pyasn1==0.4.8 # via pyasn1-modules, rsa +pyglet==1.5.0 # via gym +pygments==2.6.1 # via ipython, nbconvert +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema +python-dateutil==2.8.1 # via bokeh, jupyter-client, pandas +pytz==2020.1 # via pandas +pyyaml==5.3.1 # via bokeh, keras +pyzmq==19.0.1 # via jupyter-client, notebook +requests-oauthlib==1.3.0 # via google-auth-oauthlib +requests==2.24.0 # via requests-oauthlib, tensorboard +rsa==4.6 # via google-auth +scipy==1.4.1 # via -r requirements.in, gym, keras, tensorflow +send2trash==1.5.0 # via notebook +six==1.15.0 # via absl-py, astunparse, bleach, google-auth, google-pasta, grpcio, h5py, jsonschema, keras-preprocessing, packaging, protobuf, pyrsistent, python-dateutil, tensorboard, tensorflow, traitlets +tensorboard-plugin-wit==1.7.0 # via tensorboard +tensorboard==2.2.2 # via tensorflow +tensorflow-estimator==2.2.0 # via tensorflow +tensorflow==2.2.0 # via -r requirements.in +termcolor==1.1.0 # via tensorflow +terminado==0.8.3 # via notebook +testpath==0.4.4 # via nbconvert +tornado==6.0.4 # via bokeh, ipykernel, jupyter-client, notebook, terminado +traitlets==4.3.3 # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook +typing-extensions==3.7.4.2 # via bokeh +urllib3==1.25.9 # via requests +wcwidth==0.2.5 # via prompt-toolkit +webencodings==0.5.1 # via bleach +werkzeug==1.0.1 # via tensorboard +wheel==0.34.2 # via astunparse, tensorboard, tensorflow +widgetsnbextension==2.0.1 # via ipywidgets +wrapt==1.12.1 # via tensorflow + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From ca5b1e4df8684de60ef0077cd996d292f36f1718 Mon Sep 17 00:00:00 2001 From: krfricke Date: Thu, 23 Jul 2020 04:05:02 +0200 Subject: [PATCH 3/9] Tune exercise 3#: Fix small typo resulting in import error (#179) Co-authored-by: Kai Fricke --- tune_exercises/exercise_3_pbt.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tune_exercises/exercise_3_pbt.ipynb b/tune_exercises/exercise_3_pbt.ipynb index c0b334a..d46e73e 100644 --- a/tune_exercises/exercise_3_pbt.ipynb +++ b/tune_exercises/exercise_3_pbt.ipynb @@ -68,7 +68,7 @@ "from ray import tune\n", "from ray.tune import track\n", "from ray.tune.schedulers import PopulationBasedTraining\n", - "from ray.tune.util import validate_save_restore\n", + "from ray.tune.utils import validate_save_restore\n", "\n", "%matplotlib inline\n", "import matplotlib.style as style\n", @@ -280,4 +280,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file From 0f8adab0c252fcd94576dc23c740a6e51818134e Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Wed, 22 Jul 2020 19:05:35 -0700 Subject: [PATCH 4/9] update (#182) --- tune_exercises/exercise_1_basics.ipynb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tune_exercises/exercise_1_basics.ipynb b/tune_exercises/exercise_1_basics.ipynb index 5ce7585..e13792b 100644 --- a/tune_exercises/exercise_1_basics.ipynb +++ b/tune_exercises/exercise_1_basics.ipynb @@ -36,7 +36,7 @@ "\n", "# print(\"Setting up colab environment\")\n", "# !pip uninstall -y -q pyarrow\n", - "# !pip install -q https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev5-cp36-cp36m-manylinux1_x86_64.whl\n", + "# !pip install -q -U ray[tune]\n", "# !pip install -q ray[debug]\n", "\n", "# # A hack to force the runtime to restart, needed to include the above dependencies.\n", @@ -263,7 +263,7 @@ "outputs": [], "source": [ "import tensorflow.keras as keras\n", - "from ray.tune import track\n", + "from ray import tune\n", "\n", "\n", "class TuneReporterCallback(keras.callbacks.Callback):\n", @@ -278,7 +278,7 @@ "\n", " def on_epoch_end(self, batch, logs={}):\n", " self.iteration += 1\n", - " track.log(keras_info=logs, mean_accuracy=logs.get(\"accuracy\"), mean_loss=logs.get(\"loss\"))\n" + " tune.report(keras_info=logs, mean_accuracy=logs.get(\"accuracy\"), mean_loss=logs.get(\"loss\"))\n" ] }, { @@ -329,10 +329,10 @@ " \n", "assert len(inspect.getargspec(tune_iris).args) == 1, \"The `tune_iris` function needs to take in the arg `config`.\"\n", "\n", - "print(\"Test-running to make sure this function will run correctly.\")\n", - "tune.track.init() # For testing purposes only.\n", - "tune_iris({\"lr\": 0.1, \"dense_1\": 4, \"dense_2\": 4})\n", - "print(\"Success!\")" + "# print(\"Test-running to make sure this function will run correctly.\")\n", + "# tune.track.init() # For testing purposes only.\n", + "# tune_iris({\"lr\": 0.1, \"dense_1\": 4, \"dense_2\": 4})\n", + "# print(\"Success!\")" ] }, { @@ -547,7 +547,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.4" } }, "nbformat": 4, From 329cca731989938a1c43775bb85d71ad5aae47db Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Thu, 23 Jul 2020 09:10:50 -0700 Subject: [PATCH 5/9] update-exercise-2 --- tune_exercises/exercise_2_optimize.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tune_exercises/exercise_2_optimize.ipynb b/tune_exercises/exercise_2_optimize.ipynb index a73c061..f20bb3a 100644 --- a/tune_exercises/exercise_2_optimize.ipynb +++ b/tune_exercises/exercise_2_optimize.ipynb @@ -36,7 +36,7 @@ "\n", "# print(\"Setting up colab environment\")\n", "# !pip uninstall -y -q pyarrow\n", - "# !pip install -q https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.8.0.dev5-cp36-cp36m-manylinux1_x86_64.whl\n", + "# !pip install -q -U ray[tune]\n", "# !pip install -q ray[debug]\n", "\n", "# # A hack to force the runtime to restart, needed to include the above dependencies.\n", @@ -140,7 +140,7 @@ " for i in range(20):\n", " train(model, optimizer, train_loader) # Train for 1 epoch\n", " acc = test(model, test_loader) # Obtain validation accuracy.\n", - " # TODO: Add tune.track.log(mean_accuracy=acc) here\n", + " # TODO: Add tune.report(mean_accuracy=acc) here\n", " if i % 5 == 0:\n", " torch.save(model, \"./model.pth\") # This saves the model to the trial directory" ] @@ -530,7 +530,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.4" } }, "nbformat": 4, From 1eeb475ad8f3f192d17032f819f4ae52c2bc384e Mon Sep 17 00:00:00 2001 From: Dean Wampler Date: Mon, 3 Aug 2020 13:17:33 -0700 Subject: [PATCH 6/9] New tutorials at Anyscale Academy --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 58d1b39..7446cd7 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,9 @@ Ray Tutorial ============ +See the new Anyscale Academy tutorials at https://github.com/anyscale/academy. + + Try Ray on Google Colab ----------------------- From 35763f1e5bd1a6260ebedc56d81ee4c9252f4ff2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 15:19:37 -0700 Subject: [PATCH 7/9] Bump tensorflow from 2.2.0 to 2.2.1 (#186) Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.2.0 to 2.2.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.2.0...v2.2.1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0413797..cd23ef9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ modin==0.7.4 # via -r requirements.in nbconvert==5.6.1 # via notebook nbformat==5.0.7 # via ipywidgets, nbconvert, notebook notebook==6.0.3 # via widgetsnbextension -numpy==1.19.0 # via bokeh, gym, h5py, keras, keras-preprocessing, opencv-python, opt-einsum, pandas, scipy, tensorboard, tensorflow +numpy==1.18.5 # via bokeh, gym, h5py, keras, keras-preprocessing, opencv-python, opt-einsum, pandas, scipy, tensorboard, tensorflow oauthlib==3.1.0 # via requests-oauthlib opencv-python==4.3.0.36 # via -r requirements.in opt-einsum==3.3.0 # via tensorflow @@ -72,13 +72,13 @@ pyzmq==19.0.1 # via jupyter-client, notebook requests-oauthlib==1.3.0 # via google-auth-oauthlib requests==2.24.0 # via requests-oauthlib, tensorboard rsa==4.6 # via google-auth -scipy==1.4.1 # via -r requirements.in, gym, keras, tensorflow +scipy==1.4.1 # via -r requirements.in, gym, keras send2trash==1.5.0 # via notebook six==1.15.0 # via absl-py, astunparse, bleach, google-auth, google-pasta, grpcio, h5py, jsonschema, keras-preprocessing, packaging, protobuf, pyrsistent, python-dateutil, tensorboard, tensorflow, traitlets tensorboard-plugin-wit==1.7.0 # via tensorboard tensorboard==2.2.2 # via tensorflow tensorflow-estimator==2.2.0 # via tensorflow -tensorflow==2.2.0 # via -r requirements.in +tensorflow==2.2.1 # via -r requirements.in termcolor==1.1.0 # via tensorflow terminado==0.8.3 # via notebook testpath==0.4.4 # via nbconvert From 2c07b122f191a392ff43e7979742298b8ca3ece0 Mon Sep 17 00:00:00 2001 From: William Ma <12377941+williamma12@users.noreply.github.com> Date: Wed, 25 Nov 2020 17:11:13 -0600 Subject: [PATCH 8/9] Add nlp exercise to exercise 3 (#111) --- environment.yml | 2 + .../exercise03-Nested_Remote_Functions.ipynb | 113 ++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/environment.yml b/environment.yml index dfd909f..1aa4881 100644 --- a/environment.yml +++ b/environment.yml @@ -19,4 +19,6 @@ dependencies: - lxml - scikit-learn - setproctitle + - spacy + - wikipedia - atoma diff --git a/exercises/exercise03-Nested_Remote_Functions.ipynb b/exercises/exercise03-Nested_Remote_Functions.ipynb index 8ca3849..00cfb86 100644 --- a/exercises/exercise03-Nested_Remote_Functions.ipynb +++ b/exercises/exercise03-Nested_Remote_Functions.ipynb @@ -191,6 +191,119 @@ "source": [ "ray.timeline(filename=\"timeline03.json\")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Application: NLP Pipeline\n", + "\n", + "To demonstrate the practical applications of nested remote functions, we create a NLP pipeline that analyzes some interesting Wikipedia pages and we speed up this pipeline using nested remote functions. Though this is only a toy example, this example can easily be scaled up with very little changes! \n", + "\n", + "For this NLP pipeline, we first use the function `parse_wikipedia` to parse a Wikipedia page on a given topic. Within `parse_wikipedia`, we first tokenize the page using the `tokenize` function and then feed the tokens to `entity_recognizer`, a named entity recognizer. Not satisfied with just the named entities of each language, we also search for the Wikipedia pages of the first 10 unique named entities, recursively until we hit a recursion depth of 2. Finally, we return a pandas dataframe containing all the named entities we found.\n", + "\n", + "As an example, we parse the wikipedia entries for python, java, and c++, the languages that are used in ray!\n", + "\n", + "**NOTE:** We use the `en_core_web_sm`, a pre-trained model provided by `spacy`. Make sure you have it downloaded by running the following piece of code within a jupyter notebook cell.\n", + "\n", + "```\n", + "!python -m spacy download en_core_web_sm\n", + "```\n", + "After running the above code, you may need to restart the notebook kernel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import modin.pandas as pd\n", + "import spacy\n", + "import wikipedia\n", + "\n", + "MAX_LINKS = 2\n", + "MAX_DEPTH = 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def tokenize(text):\n", + " time.sleep(2)\n", + " nlp = spacy.load(\"en_core_web_sm\")\n", + " return nlp(text)\n", + " \n", + "def entity_recognizer(tokens, topic):\n", + " time.sleep(2)\n", + " results = []\n", + " for token in tokens.ents:\n", + " results.append([topic, token.text, token.lemma_, token.label_])\n", + " \n", + " return results\n", + "\n", + "def recursive_wiki_scraper(topic, depth=0):\n", + " try:\n", + " wiki_page = wikipedia.page(topic)\n", + " except:\n", + " return []\n", + " wiki_links = wiki_page.links[:MAX_LINKS]\n", + "\n", + " page_tokens = tokenize(wiki_page.content)\n", + " topic_result = entity_recognizer(page_tokens, topic)\n", + " result = []\n", + " \n", + " if depth < MAX_DEPTH:\n", + " for link in wiki_links:\n", + " result.extend(recursive_wiki_scraper(link, depth+1))\n", + " \n", + " result.extend(topic_result)\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's try and get some information on the languages that ray is built on!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start = time.time()\n", + "\n", + "languages = [\"Python\", \"Java programming\", \"C++\"]\n", + "results = []\n", + "for lang in languages:\n", + " results.extend(recursive_wiki_scraper(lang))\n", + " \n", + "duration = time.time() - start\n", + "print(\"Constructing the dataframe took {} seconds.\".format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Exercise:** Speed up the above NLP pipeline using ray and its nested remote functions. To do so, it is recommended that you only make `tokenize` and `entity_recognizer` be remote functions and not the `recursive_wiki_scraper`. Try and understand why this is the case. Below you should find the a sample of the results. Feel free to explore the data to find interesting associations to the programming languages that ray is written in!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(results, columns=[\"topic\", \"text\", \"lemma\", \"label\"])\n", + "df.sample(10)" + ] } ], "metadata": { From 08f4f01fc3e918c997c971f7b2421551f054c851 Mon Sep 17 00:00:00 2001 From: "Jules S. Damji" Date: Mon, 21 Mar 2022 13:43:22 -0700 Subject: [PATCH 9/9] Added deprecated message for these tutorials (#190) Signed-off-by: Jules S.Damji Co-authored-by: Jules S.Damji --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7446cd7..f6f24ea 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,8 @@ Ray Tutorial ============ -See the new Anyscale Academy tutorials at https://github.com/anyscale/academy. +**NOTE**: These sets of tutorials have been **deprecated**. A portion of their modules have +been incorporated into the new Anyscale Academy tutorials at https://github.com/anyscale/academy. Try Ray on Google Colab