From d988d7f5aa944f931a00ca3e601d1124ca3ce4e8 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 04:12:27 -0800 Subject: [PATCH 01/18] fasttext model for predict, bert test method --- .gitignore | 1 + src/Dockerfile-sentiment | 10 +- src/app/apis/SentimentV1/__init__.py | 1 + src/app/apis/SentimentV1/sentimentV1.py | 29 +++ .../sentimentV1_transfer_retraining.py | 137 +++++++++++- .../SentimentV1/sentiment_infer_server.py | 23 +- src/app/tasks_nlp.py | 26 +++ src/install_base_model.py | 21 ++ src/requirements.txt | 1 - src/requirementssenti.txt | 207 +++++++----------- 10 files changed, 313 insertions(+), 143 deletions(-) diff --git a/.gitignore b/.gitignore index 05e8229..641b4ef 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ __pycache__/ # model files *.h5 +*.bin # Distribution / packaging .Python diff --git a/src/Dockerfile-sentiment b/src/Dockerfile-sentiment index 5b5c09a..9932b1a 100644 --- a/src/Dockerfile-sentiment +++ b/src/Dockerfile-sentiment @@ -1,13 +1,13 @@ -FROM continuumio/miniconda3:4.5.12 +FROM tensorflow/tensorflow:latest-gpu # utils RUN apt-get update && apt-get install -y --no-install-recommends apt-utils -RUN conda install gxx_linux-64 +#RUN conda install gxx_linux-64 -RUN conda install python=3.6 +#RUN conda install python=3.6 -RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential +#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential # Grab requirements.txt. COPY requirementssenti.txt /tmp/requirementssenti.txt @@ -15,6 +15,8 @@ COPY requirementssenti.txt /tmp/requirementssenti.txt # Install dependencies RUN pip install -qr /tmp/requirementssenti.txt +RUN pip install fasttext + # create a user for web server RUN adduser --disabled-password --gecos "" foo diff --git a/src/app/apis/SentimentV1/__init__.py b/src/app/apis/SentimentV1/__init__.py index 1d6da5b..07b685a 100644 --- a/src/app/apis/SentimentV1/__init__.py +++ b/src/app/apis/SentimentV1/__init__.py @@ -4,3 +4,4 @@ @author: manu ''' from sentimentV1 import * +from API_helpers_nlp import * \ No newline at end of file diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py index 8d1b079..3676042 100644 --- a/src/app/apis/SentimentV1/sentimentV1.py +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -25,6 +25,8 @@ # michaniki app from ...tasks_nlp import async_train_bert +SENTIMENT_TEXT_QUEUE = app.config['SENTIMENT_TEXT_QUEUE'] +CLIENT_SLEEP = app.config['CLIENT_SLEEP'] # temp folder save image files downloaded from S3 TEMP_FOLDER = os.path.join('./tmp') @@ -101,3 +103,30 @@ def run_train_bert(): "task_id": this_id, "status": "Retraining and Fine-Tuning usign BERT is Initiated" }), 200 + +@blueprint.route('/trainbert', methods=['POST']) +def run_test_bert(): + """ + Test sentences using BERT fine tuned model + """ + s3_bucket_name = request.form.get('test_bucket_name') + model_name = request.form.get('model_name') + local_data_path = os.path.join('./tmp') + s3_bucket_prefix = '' + batch_size = 32 + nb_epoch = 3 + + # create a celer task id + this_id = celery.uuid() + + async_test_bert.apply_async((model_name, + local_data_path, + s3_bucket_name, + s3_bucket_prefix, + nb_epoch, + batch_size, + this_id), task_id=this_id) + return jsonify({ + "task_id": this_id, + "status": "Retraining and Fine-Tuning usign BERT is Initiated" + }), 200 \ No newline at end of file diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 6bf6965..da9ff9d 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -1,11 +1,13 @@ ''' Created on Jan 13, 2018 -@author: manu +Most of the code is taken from run_classifier.py from +the Google BERT github - https://github.com/google-research/bert ''' import os import glob import logging +import boto3 #Import BERT related file - USED as is from Google/BERT import modeling @@ -162,7 +164,138 @@ def traineval_model(self, local_dir, for key in sorted(result.keys()): tf.logging.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) - + return result + + def test_model(self, local_dir, + nb_epoch, + batch_size, + bucket_name): + """ + Use the BERT Uncased language model to train on + new data + """ + tf.logging.set_verbosity(tf.logging.INFO) + logging.info("*:BERT MODEL PATH:%s",BERT_MODEL_PATH) + logging.info("*:Local Dir%s",local_dir) + + + mod_name = self.model_name + BERT_MODEL = 'uncased_L-12_H-768_A-12' + BERT_PRETRAINED_DIR = BERT_MODEL_PATH + OUTPUT_DIR = os.path.join(local_dir,'output_bert') + DATA_DIR = os.path.join(local_dir,'data') + logging.info('***** Model output directory: %s*****',OUTPUT_DIR) + logging.info('***** BERT pretrained directory: %s *****',BERT_PRETRAINED_DIR) + logging.info('***** DATA directory: %s *****',DATA_DIR) + TRAIN_BATCH_SIZE = 32 + EVAL_BATCH_SIZE = 8 + LEARNING_RATE = 2e-5 + NUM_TRAIN_EPOCHS = 3.0 + WARMUP_PROPORTION = 0.1 + MAX_SEQ_LENGTH = 128 + # Model configs + # if you wish to finetune a model on a larger dataset, use larger interval + SAVE_CHECKPOINTS_STEPS = 1000 + # each checpoint weights about 1,5gb + ITERATIONS_PER_LOOP = 1000 + NUM_TPU_CORES = 8 + + VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR,'vocab.txt') + BERT_CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR,'bert_config.json') + with open(os.path.join(OUTPUT_DIR,'final_ckpt.txt')) as f: + content = f.readlines() + logging.info("***Final_cktp->%s\n",content) + test_ckpt = content.split('/')[-1] + INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, test_ckpt) + DO_LOWER_CASE = BERT_MODEL.startswith('uncased') + + logging.info("Found VOCAB File:%s",VOCAB_FILE) + bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) + tf.gfile.MakeDirs(OUTPUT_DIR) + processor = run_classifier.ColaProcessor() + label_list = processor.get_labels() + tokenizer = tokenization.FullTokenizer( + vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) + + # Since training will happen on GPU, we won't need a cluster resolver + tpu_cluster_resolver = None + # TPUEstimator also supports training on CPU and GPU. You don't need to define a separate tf.estimator.Estimator. + run_config = tf.contrib.tpu.RunConfig( + cluster=tpu_cluster_resolver, + model_dir=OUTPUT_DIR, + save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS, + tpu_config=tf.contrib.tpu.TPUConfig( + iterations_per_loop=ITERATIONS_PER_LOOP, + num_shards=NUM_TPU_CORES, + per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2)) + + train_examples = None + num_train_steps = None + num_warmup_steps = None + train_examples = processor.get_train_examples(DATA_DIR) + num_train_steps = int( + len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS) + num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION) + + model_fn = run_classifier.model_fn_builder( + bert_config=bert_config, + num_labels=len(label_list), + init_checkpoint=INIT_CHECKPOINT, + learning_rate=LEARNING_RATE, + num_train_steps=num_train_steps, + num_warmup_steps=num_warmup_steps, + use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available + use_one_hot_embeddings=False) #Try with True + + estimator = tf.contrib.tpu.TPUEstimator( + use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available + model_fn=model_fn, + config=run_config, + train_batch_size=TRAIN_BATCH_SIZE, + eval_batch_size=EVAL_BATCH_SIZE) + + predict_examples = processor.get_test_examples(DATA_DIR) + num_actual_predict_examples = len(predict_examples) + predict_file = os.path.join(OUTPUT_DIR, "predict.tf_record") + run_classifier.file_based_convert_examples_to_features(predict_examples, label_list, + MAX_SEQ_LENGTH, tokenizer, + predict_file) + + tf.logging.info("***** Running prediction*****") + tf.logging.info(" Num examples = %d (%d actual, %d padding)", + len(predict_examples), num_actual_predict_examples, + len(predict_examples) - num_actual_predict_examples) + tf.logging.info(" Batch size = %d", batch_size) + + predict_input_fn = run_classifier.file_based_input_fn_builder( + input_file=predict_file, + seq_length=MAX_SEQ_LENGTH, + is_training=False, + drop_remainder=False) + + result = estimator.predict(input_fn=predict_input_fn) + output_predict_file = os.path.join(OUTPUT_DIR, "test_results.tsv") + with tf.gfile.GFile(output_predict_file, "w") as writer: + num_written_lines = 0 + tf.logging.info("***** Predict results *****") + for (i, prediction) in enumerate(result): + probabilities = prediction["probabilities"] + if i >= num_actual_predict_examples: + break + output_line = "\t".join( + str(class_probability) + for class_probability in probabilities) + "\n" + writer.write(output_line) + num_written_lines += 1 + assert num_written_lines == num_actual_predict_examples + s3 = boto3.resource('s3') + try: + s3.upload_file(output_predict_file, bucket_name, output_predict_file) + except Exception as err: + logging.info("Unable to upload to S3") + logging.info(err) + + return 1 diff --git a/src/app/models/SentimentV1/sentiment_infer_server.py b/src/app/models/SentimentV1/sentiment_infer_server.py index e2cb3bd..df4ca79 100644 --- a/src/app/models/SentimentV1/sentiment_infer_server.py +++ b/src/app/models/SentimentV1/sentiment_infer_server.py @@ -7,10 +7,10 @@ import redis import time import json -from textblob import TextBlob import logging from collections import defaultdict - +import fasttext +import re #helpers import settings @@ -22,6 +22,11 @@ class sentimentV1_inference_server: def __init__(self): # pre-load some models here on start self.loaded_models = {} + + def strip_formatting(self, string): + string = string.lower() + string = re.sub(r"([.!?,'/()])", r" \1 ", string) + return string def run_sentimentV1_infernece_server(self): ''' @@ -33,6 +38,9 @@ def run_sentimentV1_infernece_server(self): Sentecnes are tracked using is their id ''' logging.info("Sentiment Inference Server running") + FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin") + logging.info("IMDB Model path:%s",FAST_IMDB_MODEL_PATH) + classifier = fasttext.load_model(FAST_IMDB_MODEL_PATH) while True: queue = db.lrange(settings.TEXT_QUEUE, 0, settings.BATCH_SIZE) #Is this queue different from the Queue in API path textIDs = defaultdict(list) #dict to hold sentence and id for a model type @@ -58,11 +66,16 @@ def run_sentimentV1_infernece_server(self): logging.info("* Predicting for {} of Models".format(len(textIDs.keys()))) logging.info("* Number of Sentences: {}".format(num_text)) - + reviews=[] for t in text_list: logging.info("Text is:%s",t["text"]) - preds = TextBlob(t["text"]) - res = {"polarity":preds.sentiment.polarity,"subjectvity":preds.sentiment.subjectivity} + reviews.append(t["text"]) + preprocessed_reviews = list(map(self.strip_formatting, reviews)) + result = classifier.predict_proba(preprocessed_reviews, 1) + if result[0][0][0] == '__label__1': + label = 'positive' + else: label = 'negative' + res = {"label":label,"probability":result[0][0][1]} db.set(t["id"], json.dumps(res)) db.ltrim(settings.TEXT_QUEUE, len(textIDs), -1) diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py index 3c58a59..fb4da10 100644 --- a/src/app/tasks_nlp.py +++ b/src/app/tasks_nlp.py @@ -50,3 +50,29 @@ def async_train_bert(model_name, logging.info(err) #shutil.rmtree(text_data_path, ignore_errors=True) raise + +def async_test_bert(model_name, + local_data_path, + s3_bucket_name, + s3_bucket_prefix, + nb_epoch, + batch_size, + id): + """ + train a model using BERT pre-trained model + """ + text_data_path = API_helpers_nlp.download_a_dir_from_s3(s3_bucket_name, + s3_bucket_prefix, + local_path = TEMP_FOLDER) + + logging.info('*Text Data Path:%s',text_data_path) + try: + bert_transfer = sentimentV1_transfer_retraining.BertTransferLeaner(model_name) + new_model_eval_res = bert_transfer.test_model(text_data_path,nb_epoch,batch_size,s3_bucket_name) + logging.info("****Test done, file saved in S3") + print(new_model_eval_res) + return str(new_model_eval_res['eval_accuracy']),str(new_model_eval_res['global_step']) + except Exception as err: + logging.info(err) + #shutil.rmtree(text_data_path, ignore_errors=True) + raise diff --git a/src/install_base_model.py b/src/install_base_model.py index 031c6bb..2d065a6 100644 --- a/src/install_base_model.py +++ b/src/install_base_model.py @@ -5,12 +5,16 @@ import zipfile, io from tqdm import tqdm import math +import boto3 +import botocore from keras.applications.inception_v3 import InceptionV3 BASE_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "base", "base.h5") TOPLESS_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "topless") BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt.data-00000-of-00001") BERT_DIR_PATH = os.path.join("app", "models", "SentimentV1") +FAST_IMDB_DIR = os.path.join("app", "models", "SentimentV1","fastimdb") +FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin") # loading base model if os.path.exists(BASE_MODEL_PATH): @@ -48,6 +52,23 @@ z = zipfile.ZipFile(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip')) z.extractall(BERT_DIR_PATH) + os.remove(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip')) + +if os.path.exists(FAST_IMDB_MODEL_PATH): + logging.info("* Found Base IMDB model") +else: + logging.info("Base IMDB model not found. Downloading....") + s3 = boto3.resource('s3') + mybucket = s3.Bucket('fastimdb') + model_name = 'imdb_model.bin' + try: + os.makedirs(FAST_IMDB_DIR) + except OSError: + pass + try: + mybucket.download_file(model_name, os.path.join(FAST_IMDB_DIR,'imdb_model.bin')) + except Exception as err: + logging.info(err) # clean up the died images upon start: # need to wait a bit for redis container to start up diff --git a/src/requirements.txt b/src/requirements.txt index da80471..9c13285 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -87,7 +87,6 @@ singledispatch==3.4.0.3 six==1.11.0 subprocess32==3.5.1 tensorboard==1.8.0 -tensorflow-gpu==1.11.0 termcolor==1.1.0 tornado==5.0.2 tqdm==4.26.0 diff --git a/src/requirementssenti.txt b/src/requirementssenti.txt index a889dbe..95ab373 100644 --- a/src/requirementssenti.txt +++ b/src/requirementssenti.txt @@ -1,154 +1,99 @@ -absl-py==0.6.1 -argh==0.26.2 +absl-py==0.2.2 +appnope==0.1.0 asn1crypto==0.24.0 -astor==0.7.1 -backcall==0.1.0 +astor==0.6.2 +aws==0.2.5 +awscli==1.15.38 backports-abc==0.5 -base58==1.0.3 -beautifulsoup4==4.6.3 -bleach==3.0.2 -boto==2.49.0 -boto3==1.9.78 -botocore==1.12.78 -Bottleneck==1.2.1 -bz2file==0.98 -certifi==2018.11.29 +backports.functools-lru-cache==1.5 +backports.shutil-get-terminal-size==1.0.0 +backports.weakref==1.0.post1 +bcdoc==0.12.2 +bcrypt==3.1.4 +bleach==1.5.0 +boto==2.48.0 +boto3==1.7.38 +botocore==1.10.38 +certifi==2018.4.16 +celery==4.2.0 cffi==1.11.5 chardet==3.0.4 -Click==7.0 -cryptography==2.4.2 +click==6.7 +colorama==0.2.5 +cryptography==2.2.2 cycler==0.10.0 -cymem==2.0.2 -cytoolz==0.9.0.1 -dataclasses==0.6 +cython==0.29.4 decorator==4.3.0 -Deprecated==1.2.4 -dill==0.2.8.2 +Django==1.11.15 docutils==0.14 -entrypoints==0.2.3 -enum-compat==0.0.2 -fastai==1.0.40 -fastprogress==0.1.18 -flair==0.4.0 +enum34==1.1.6 +envparse==0.2.0 +fabric==2.1.3 Flask==1.0.2 -future==0.17.1 -future-fstrings==0.4.5 -futures==3.1.1 -gast==0.2.2 -gensim==3.4.0 -gluonnlp==0.5.0.post0 -graphviz==0.8.4 -grpcio==1.17.1 -h5py==2.9.0 -hyperopt==0.1.1 -idna==2.8 -ipykernel==5.1.0 -ipython==7.2.0 +funcsigs==1.0.2 +futures==3.2.0 +gast==0.2.0 +grpcio==1.12.1 +h5py==2.8.0 +html5lib==0.9999999 +idna==2.7 +image==1.5.24 +invoke==1.0.0 +ipaddress==1.0.22 +ipykernel==4.8.2 +ipython==5.7.0 ipython-genutils==0.2.0 -ipywidgets==7.4.2 -itsdangerous==1.1.0 -jedi==0.13.2 +itsdangerous==0.24 Jinja2==2.10 jmespath==0.9.3 -jsonschema==2.6.0 -jupyter==1.0.0 -jupyter-client==5.2.4 -jupyter-console==6.0.0 +jupyter-client==5.2.3 jupyter-core==4.4.0 -Keras==2.2.4 -Keras-Applications==1.0.6 -Keras-Preprocessing==1.0.5 +Keras==2.0.0 +Keras-Applications==1.0.2 +Keras-Preprocessing==1.0.1 kiwisolver==1.0.1 -Markdown==3.0.1 -MarkupSafe==1.1.0 -matplotlib==3.0.0 -mistune==0.8.4 -mkl-fft==1.0.6 -mkl-random==1.0.1 +Markdown==2.6.11 +MarkupSafe==1.0 +matplotlib==2.2.2 mock==2.0.0 -mpld3==0.3 -msgpack==0.5.6 -msgpack-numpy==0.4.3.2 -murmurhash==1.0.1 -mxnet-cu90==1.3.1 -mypy==0.650 -mypy-extensions==0.4.1 -networkx==2.2 -nltk==3.4 -notebook==5.7.4 -numexpr==2.6.8 -numpy==1.14.6 -olefile==0.46 -packaging==18.0 -pandas==0.23.4 -pandocfilters==1.4.2 -parso==0.3.1 -pathtools==0.1.2 -pbr==5.1.1 +numpy==1.14.4 +paramiko==2.4.2 +pathlib2==2.3.2 +pbr==4.0.4 pexpect==4.6.0 -pickleshare==0.7.5 -Pillow==5.4.1 -plac==0.9.6 -preshed==2.0.1 -prometheus-client==0.5.0 -prompt-toolkit==2.0.7 -protobuf==3.6.1 -psutil==5.4.8 -ptyprocess==0.6.0 -pycparser==2.19 -Pygments==2.3.1 -pymongo==3.7.2 -pyOpenSSL==18.0.0 -pyparsing==2.3.0 -PySocks==1.6.8 -python-dateutil==2.7.5 -pytorch-pretrained-bert==0.3.0 -pytz==2018.7 -PyYAML==3.13 -pyzmq==17.1.2 -qtconsole==4.4.3 -redis==3.0.1 -regex==2018.1.10 -requests==2.21.0 +pickleshare==0.7.4 +Pillow==5.1.0 +prettytable==0.7.2 +prompt-toolkit==1.0.15 +protobuf==3.6.0 +ptyprocess==0.5.2 +pyasn1==0.4.3 +pycparser==2.18 +Pygments==2.2.0 +PyNaCl==1.2.1 +pyparsing==2.2.0 +python-dateutil==2.7.3 +pytz==2018.4 +PyYAML==3.12 +pyzmq==17.0.0 +redis==2.10.6 +requests==2.19.0 +rq==0.11.0 +rsa==3.1.2 s3transfer==0.1.13 -scikit-learn==0.20.1 +scandir==1.7 scipy==1.1.0 -seaborn==0.9.0 -segtok==1.5.7 -Send2Trash==1.5.0 +simplegeneric==0.8.1 singledispatch==3.4.0.3 -six==1.12.0 -sklearn==0.0 -smart-open==1.8.0 -spacy==2.0.18 -sqlitedict==1.6.0 -streamlit==0.23.0 -tensorboard==1.12.2 -tensorflow==1.12.0 -tensorflow-hub==0.2.0 +six==1.11.0 +subprocess32==3.5.1 +tensorboard==1.8.0 +tensorflow-gpu==1.11.0 termcolor==1.1.0 -terminado==0.8.1 -testpath==0.4.2 -textblob==0.15.2 -thinc==6.12.1 -tokenize-rt==2.1.0 -toml==0.10.0 -toolz==0.9.0 -torch==1.0.0 -torchtext==0.3.1 -torchvision==0.2.1 -tornado==5.1.1 +tornado==5.0.2 tqdm==4.26.0 traitlets==4.3.2 -typed-ast==1.1.1 -typing==3.6.4 -tzlocal==1.5.1 -ujson==1.35 -urllib3==1.24.1 -watchdog==0.9.0 +urllib3==1.23 +uWSGI==2.0.17 wcwidth==0.1.7 -webencodings==0.5.1 Werkzeug==0.14.1 -widgetsnbextension==3.4.2 -wrapt==1.10.11 -xgboost==0.81 From 53a4e4914f89abee6662879f3699fcc13373d69c Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 11:14:45 -0800 Subject: [PATCH 02/18] fasttext model for predict load in loop, bert test method --- src/app/models/SentimentV1/sentiment_infer_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/models/SentimentV1/sentiment_infer_server.py b/src/app/models/SentimentV1/sentiment_infer_server.py index df4ca79..e786cf7 100644 --- a/src/app/models/SentimentV1/sentiment_infer_server.py +++ b/src/app/models/SentimentV1/sentiment_infer_server.py @@ -40,7 +40,7 @@ def run_sentimentV1_infernece_server(self): logging.info("Sentiment Inference Server running") FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin") logging.info("IMDB Model path:%s",FAST_IMDB_MODEL_PATH) - classifier = fasttext.load_model(FAST_IMDB_MODEL_PATH) + while True: queue = db.lrange(settings.TEXT_QUEUE, 0, settings.BATCH_SIZE) #Is this queue different from the Queue in API path textIDs = defaultdict(list) #dict to hold sentence and id for a model type @@ -63,6 +63,7 @@ def run_sentimentV1_infernece_server(self): if textIDs: + classifier = fasttext.load_model(FAST_IMDB_MODEL_PATH) logging.info("* Predicting for {} of Models".format(len(textIDs.keys()))) logging.info("* Number of Sentences: {}".format(num_text)) From aff64419c44a4d7eeb1d28355b6f914094487791 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 11:57:05 -0800 Subject: [PATCH 03/18] add print for debug --- src/install_base_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/install_base_model.py b/src/install_base_model.py index 2d065a6..a74f5bd 100644 --- a/src/install_base_model.py +++ b/src/install_base_model.py @@ -11,7 +11,7 @@ BASE_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "base", "base.h5") TOPLESS_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "topless") -BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt.data-00000-of-00001") +BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt") BERT_DIR_PATH = os.path.join("app", "models", "SentimentV1") FAST_IMDB_DIR = os.path.join("app", "models", "SentimentV1","fastimdb") FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin") @@ -55,7 +55,7 @@ os.remove(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip')) if os.path.exists(FAST_IMDB_MODEL_PATH): - logging.info("* Found Base IMDB model") + print "* Found Base IMDB model" else: logging.info("Base IMDB model not found. Downloading....") s3 = boto3.resource('s3') From 60be236723781b2fca62da2819fb856074c5e2ee Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 12:18:15 -0800 Subject: [PATCH 04/18] fix bug in test bert --- src/app/apis/SentimentV1/sentimentV1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py index 3676042..0e7ec10 100644 --- a/src/app/apis/SentimentV1/sentimentV1.py +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -104,7 +104,7 @@ def run_train_bert(): "status": "Retraining and Fine-Tuning usign BERT is Initiated" }), 200 -@blueprint.route('/trainbert', methods=['POST']) +@blueprint.route('/testbert', methods=['POST']) def run_test_bert(): """ Test sentences using BERT fine tuned model From c8a97aef5b1eed4c476e842d8dae82b6b014af47 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 12:20:29 -0800 Subject: [PATCH 05/18] fix bug 2 in test bert --- src/app/apis/SentimentV1/sentimentV1.py | 1 + src/app/tasks_nlp.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py index 0e7ec10..03f0508 100644 --- a/src/app/apis/SentimentV1/sentimentV1.py +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -24,6 +24,7 @@ # michaniki app from ...tasks_nlp import async_train_bert +from ...tasks_nlp import async_test_bert SENTIMENT_TEXT_QUEUE = app.config['SENTIMENT_TEXT_QUEUE'] CLIENT_SLEEP = app.config['CLIENT_SLEEP'] diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py index fb4da10..7fa2198 100644 --- a/src/app/tasks_nlp.py +++ b/src/app/tasks_nlp.py @@ -51,6 +51,7 @@ def async_train_bert(model_name, #shutil.rmtree(text_data_path, ignore_errors=True) raise +@michaniki_celery_app.task() def async_test_bert(model_name, local_data_path, s3_bucket_name, From ffbe4559e188ebd0a51d2e2f603cd989d6d5707f Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 12:28:26 -0800 Subject: [PATCH 06/18] added new method to download test file --- src/app/apis/SentimentV1/API_helpers_nlp.py | 37 ++++++++++++++++++++- src/app/tasks_nlp.py | 2 +- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py index 72bf292..46544b8 100644 --- a/src/app/apis/SentimentV1/API_helpers_nlp.py +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -42,7 +42,7 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): logging.info('*Saving text files at:%s',save_path) s3 = boto3.resource('s3') - mybucket = s3.Bucket(bucket_name) + # if blank prefix is given, return everything) key1 = 'train.tsv' key2 = 'val.tsv' @@ -54,6 +54,41 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): try: s3.Bucket(bucket_name).download_file(key1, os.path.join(save_path,'train.tsv')) s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'dev.tsv')) + s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'test.tsv')) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "404": + print("The object does not exist.") + else: + raise + + logging.info("* Helper: Text Loaded at: {}".format(output_path)) + return output_path + +def download_test_file_from_s3(bucket_name, bucket_prefix, local_path): + """ + download the folder from S3 + + local: /src/tmp/model_data/ + + Will not download if the local folder already exists + """ + logging.info("* Helper: Loading Text from S3 {} {}".format(bucket_name,bucket_prefix)) + path = os.path.join(bucket_name,'data') + output_path = os.path.join(local_path, bucket_name) + save_path = os.path.join(local_path, path) + logging.info('*Saving text files at:%s',save_path) + + s3 = boto3.resource('s3') + mybucket = s3.Bucket(bucket_name) + # if blank prefix is given, return everything) + + key3 = 'test.tsv' + try: + os.makedirs(save_path) + except OSError: + pass + try: + mybucket.download_file(key2, os.path.join(save_path,'test.tsv')) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": print("The object does not exist.") diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py index 7fa2198..b1ab4b1 100644 --- a/src/app/tasks_nlp.py +++ b/src/app/tasks_nlp.py @@ -62,7 +62,7 @@ def async_test_bert(model_name, """ train a model using BERT pre-trained model """ - text_data_path = API_helpers_nlp.download_a_dir_from_s3(s3_bucket_name, + text_data_path = API_helpers_nlp.download_test_file_from_s3(s3_bucket_name, s3_bucket_prefix, local_path = TEMP_FOLDER) From 0741cb4b755449d41eafaae9dbd498e1f0928d99 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 12:59:21 -0800 Subject: [PATCH 07/18] fix bug in method to download test file --- src/app/apis/SentimentV1/API_helpers_nlp.py | 4 ++-- src/app/apis/SentimentV1/sentimentV1.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py index 46544b8..b253edd 100644 --- a/src/app/apis/SentimentV1/API_helpers_nlp.py +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -54,7 +54,7 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): try: s3.Bucket(bucket_name).download_file(key1, os.path.join(save_path,'train.tsv')) s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'dev.tsv')) - s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'test.tsv')) + except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": print("The object does not exist.") @@ -88,7 +88,7 @@ def download_test_file_from_s3(bucket_name, bucket_prefix, local_path): except OSError: pass try: - mybucket.download_file(key2, os.path.join(save_path,'test.tsv')) + mybucket.download_file(key3, os.path.join(save_path,'test.tsv')) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": print("The object does not exist.") diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py index 03f0508..1ea6c79 100644 --- a/src/app/apis/SentimentV1/sentimentV1.py +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -129,5 +129,5 @@ def run_test_bert(): this_id), task_id=this_id) return jsonify({ "task_id": this_id, - "status": "Retraining and Fine-Tuning usign BERT is Initiated" + "status": "Testing usign BERT is Started. Results uploaded to S3 bucket" }), 200 \ No newline at end of file From d95e683936c184996cbc466c86e78c31deb8968d Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 13:18:02 -0800 Subject: [PATCH 08/18] fix bug test bert method --- .../models/SentimentV1/sentimentV1_transfer_retraining.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index da9ff9d..64171af 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -190,6 +190,7 @@ def test_model(self, local_dir, logging.info('***** DATA directory: %s *****',DATA_DIR) TRAIN_BATCH_SIZE = 32 EVAL_BATCH_SIZE = 8 + PREDICT_BATCH_SIZE = 32 LEARNING_RATE = 2e-5 NUM_TRAIN_EPOCHS = 3.0 WARMUP_PROPORTION = 0.1 @@ -206,7 +207,7 @@ def test_model(self, local_dir, with open(os.path.join(OUTPUT_DIR,'final_ckpt.txt')) as f: content = f.readlines() logging.info("***Final_cktp->%s\n",content) - test_ckpt = content.split('/')[-1] + test_ckpt = content[0].split('/')[-1] INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, test_ckpt) DO_LOWER_CASE = BERT_MODEL.startswith('uncased') @@ -253,7 +254,8 @@ def test_model(self, local_dir, model_fn=model_fn, config=run_config, train_batch_size=TRAIN_BATCH_SIZE, - eval_batch_size=EVAL_BATCH_SIZE) + eval_batch_size=EVAL_BATCH_SIZE, + predict_batch_size=PREDICT_BATCH_SIZE) predict_examples = processor.get_test_examples(DATA_DIR) num_actual_predict_examples = len(predict_examples) From 79097e4f588ce7ebecdba936f45df6fe4862ecb6 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 13:31:26 -0800 Subject: [PATCH 09/18] fix bug in test bert method --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 64171af..d4b3864 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -208,7 +208,7 @@ def test_model(self, local_dir, content = f.readlines() logging.info("***Final_cktp->%s\n",content) test_ckpt = content[0].split('/')[-1] - INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, test_ckpt) + INIT_CHECKPOINT = os.path.join(OUTPUT_DIR, test_ckpt) DO_LOWER_CASE = BERT_MODEL.startswith('uncased') logging.info("Found VOCAB File:%s",VOCAB_FILE) @@ -234,7 +234,7 @@ def test_model(self, local_dir, train_examples = None num_train_steps = None num_warmup_steps = None - train_examples = processor.get_train_examples(DATA_DIR) + train_examples = processor.get_test_examples(DATA_DIR) num_train_steps = int( len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS) num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION) From 4e8c3f3244512466cd72074fc5a4a65e8617a9d2 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 13:45:06 -0800 Subject: [PATCH 10/18] add tf-gpu to requirements --- .../models/SentimentV1/sentimentV1_transfer_retraining.py | 5 +---- src/requirements.txt | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index d4b3864..5e11c83 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -234,10 +234,7 @@ def test_model(self, local_dir, train_examples = None num_train_steps = None num_warmup_steps = None - train_examples = processor.get_test_examples(DATA_DIR) - num_train_steps = int( - len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS) - num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION) + model_fn = run_classifier.model_fn_builder( bert_config=bert_config, diff --git a/src/requirements.txt b/src/requirements.txt index 9c13285..da80471 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -87,6 +87,7 @@ singledispatch==3.4.0.3 six==1.11.0 subprocess32==3.5.1 tensorboard==1.8.0 +tensorflow-gpu==1.11.0 termcolor==1.1.0 tornado==5.0.2 tqdm==4.26.0 From db12032fc28d21bf86be953cab603aa41b2c1750 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 17:26:30 -0800 Subject: [PATCH 11/18] add log statement --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 5e11c83..4278af3 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -289,6 +289,7 @@ def test_model(self, local_dir, num_written_lines += 1 assert num_written_lines == num_actual_predict_examples s3 = boto3.resource('s3') + tf.logging.info("Done with prediction uploading results to S3") try: s3.upload_file(output_predict_file, bucket_name, output_predict_file) except Exception as err: From 9665d5f83f95337476509e9a596bf9fc7775ff34 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 18:22:45 -0800 Subject: [PATCH 12/18] fix s3 upload error statement --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 2 +- src/app/tasks_nlp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 4278af3..4bdbc96 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -291,7 +291,7 @@ def test_model(self, local_dir, s3 = boto3.resource('s3') tf.logging.info("Done with prediction uploading results to S3") try: - s3.upload_file(output_predict_file, bucket_name, output_predict_file) + s3.Bucket(bucket_name).upload_file(output_predict_file, output_predict_file) except Exception as err: logging.info("Unable to upload to S3") logging.info(err) diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py index b1ab4b1..f476677 100644 --- a/src/app/tasks_nlp.py +++ b/src/app/tasks_nlp.py @@ -72,7 +72,7 @@ def async_test_bert(model_name, new_model_eval_res = bert_transfer.test_model(text_data_path,nb_epoch,batch_size,s3_bucket_name) logging.info("****Test done, file saved in S3") print(new_model_eval_res) - return str(new_model_eval_res['eval_accuracy']),str(new_model_eval_res['global_step']) + return str(1),str(1) except Exception as err: logging.info(err) #shutil.rmtree(text_data_path, ignore_errors=True) From 5658cd8f8344bdbe25fc688cdad49892d10fd3f5 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 20:27:09 -0800 Subject: [PATCH 13/18] add try block in inception v3 to fix tx learning --- src/app/apis/InceptionV3/API_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/app/apis/InceptionV3/API_helpers.py b/src/app/apis/InceptionV3/API_helpers.py index 2beef9b..180fe86 100644 --- a/src/app/apis/InceptionV3/API_helpers.py +++ b/src/app/apis/InceptionV3/API_helpers.py @@ -69,7 +69,10 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): os.makedirs(save_path) except OSError: pass - mybucket.download_file(obj.key, os.path.join(save_path, filename)) + try: + mybucket.download_file(obj.key, os.path.join(save_path, filename)) + except OSError: + pass print "* Helper: Images Loaded at: {}".format(output_path) return output_path From a4d536932adc974f488310929ae50832eea425e8 Mon Sep 17 00:00:00 2001 From: Manu Date: Sun, 3 Feb 2019 20:58:41 -0800 Subject: [PATCH 14/18] updated readme --- README.md | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2e0e24e..fd761d5 100644 --- a/README.md +++ b/README.md @@ -271,20 +271,20 @@ curl -X POST \ ### 2. Train a new classification model using pre-trained BERT model -**The new text dataset should be stored at S3 first, with the directory architecture in S3 should look like this**: +**The new text dataset should be stored at S3 first, the directory architecture in S3 should look like this**: ``` . ├── YOUR_BUCKET_NAME │ ├── train.tsv -│ ├── dev.tsv +│ ├── val.tsv │ ├── test.tsv ``` The folder name you give to *YOUR_MODEL_NAME* will be used to identify this model once it get trained. -The name of train, dev and test files **can't be changed**. +The name of train, val and test files **can't be changed**. The train and dev file should have below format (without header)- id label None Sentence -1 0 NC +1 0 NC Text The test.tsv file should only have id and sentence column (with header) **The S3 folders should have public access permission**. @@ -297,4 +297,25 @@ curl -X POST \ -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \ -F train_bucket_name=YOUR_BUCKET_NAME \ -F train_bucket_prefix=YOUR_MODEL_NAME -``` \ No newline at end of file +``` +### 3. Lable all text in a csv file using pre-trained BERT model + +**The new test tsv file should be stored at the same S3 bucket as above for that model, directory architecture in S3 should look like this**: +``` +. +├── YOUR_BUCKET_NAME +│ ├── train.tsv +│ ├── val.tsv +│ ├── test.tsv +``` +To call this API do: +```bash +curl -X POST \ + http://127.0.0.1:3031/sentimentV1/testbert \ + -H 'Cache-Control: no-cache' \ + -H 'Postman-Token: 4e90e1d6-de18-4501-a82c-f8a878616b12' \ + -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \ + -F test_bucket_name=YOUR_BUCKET_NAME \ + -F test_bucket_prefix=YOUR_MODEL_NAME +``` +At the end of prediction a file named 'test_results.csv' will be uploaded to the same S3 bucket. \ No newline at end of file From 34fa832ef509ec9e49d50d6e37635552da840337 Mon Sep 17 00:00:00 2001 From: Manu Date: Mon, 4 Feb 2019 21:20:51 -0800 Subject: [PATCH 15/18] added 2nd docker-compose-gpu.yml for gpu training, so inference now works on CPU --- docker-compose-gpu.yml | 99 +++++++++++++++++++++ docker-compose.yml | 4 - src/Dockerfile | 6 +- src/Dockerfile-gpu | 21 +++++ src/app/apis/SentimentV1/API_helpers_nlp.py | 6 +- src/app/apis/SentimentV1/sentimentV1.py | 10 +-- src/app/index.html | 22 +++++ src/app/tasks_nlp.py | 6 +- src/requirements-gpu.txt | 98 ++++++++++++++++++++ src/requirements.txt | 2 +- src/requirementssenti.txt | 2 +- 11 files changed, 253 insertions(+), 23 deletions(-) create mode 100644 docker-compose-gpu.yml create mode 100644 src/Dockerfile-gpu create mode 100644 src/app/index.html create mode 100644 src/requirements-gpu.txt diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml new file mode 100644 index 0000000..bab364c --- /dev/null +++ b/docker-compose-gpu.yml @@ -0,0 +1,99 @@ +version: '2.3' + +services: + michaniki_client: + build: + context: ./src + dockerfile: Dockerfile-gpu + runtime: nvidia + ports: + - "3031:3031" + environment: + - PORT=3031 + - FLAS_APP=app/__init__.py + - FLASK_DEBUG=1 + - REDIS_URL="redis://redis" + - REDIS_PORT=6379 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + + volumes: + - ./src:/opt/src + + command: ./entryPoint.sh + depends_on: + - redis + networks: + - michaniki + + inference_server: + build: + context: ./src + dockerfile: Dockerfile-gpu + runtime: nvidia + environment: + - REDIS_URL="redis://redis" + - REDIS_PORT=6379 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + + command: ['python', 'app/models/InceptionV3/inception_inference_server.py'] + volumes: + - ./src:/opt/src + + networks: + - michaniki + depends_on: + - michaniki_client + - redis + + sentiment_inference_server: + build: + context: ./src + dockerfile: Dockerfile-sentiment + runtime: nvidia + environment: + - REDIS_URL="redis://redis" + - REDIS_PORT=6379 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + command: ['python', 'app/models/SentimentV1/sentiment_infer_server.py'] + volumes: + - ./src:/opt/src + networks: + - michaniki + depends_on: + - michaniki_client + - redis + + celery_worker: + build: + context: ./src + dockerfile: Dockerfile-gpu + runtime: nvidia + command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] + volumes: + - ./src:/opt/src + networks: + - michaniki + depends_on: + - michaniki_client + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - DB_HOST=db + - DB_USERNAME=root + - DB_PASSWORD=michaniki + - DB_NAME=michanikidb + - BROKER_URL=redis://redis:6379/0 + + + redis: + image: redis:4.0.5-alpine + command: ["redis-server", "--appendonly", "yes"] + hostname: redis + networks: + - michaniki + +networks: + michaniki: diff --git a/docker-compose.yml b/docker-compose.yml index 6561d3c..a7d1b45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ version: '2.3' services: michaniki_client: build: ./src - runtime: nvidia ports: - "3031:3031" environment: @@ -26,7 +25,6 @@ services: inference_server: build: ./src - runtime: nvidia environment: - REDIS_URL="redis://redis" - REDIS_PORT=6379 @@ -47,7 +45,6 @@ services: build: context: ./src dockerfile: Dockerfile-sentiment - runtime: nvidia environment: - REDIS_URL="redis://redis" - REDIS_PORT=6379 @@ -64,7 +61,6 @@ services: celery_worker: build: ./src - runtime: nvidia command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] volumes: - ./src:/opt/src diff --git a/src/Dockerfile b/src/Dockerfile index 577b4d2..3c8e96e 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,11 +1,11 @@ -FROM tensorflow/tensorflow:latest-gpu +FROM continuumio/miniconda:4.4.10 # utils RUN apt-get update && apt-get install -y --no-install-recommends apt-utils -#RUN conda install gxx_linux-64 +RUN conda install gxx_linux-64 -#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential +RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential # Grab requirements.txt. COPY requirements.txt /tmp/requirements.txt diff --git a/src/Dockerfile-gpu b/src/Dockerfile-gpu new file mode 100644 index 0000000..e7dcecb --- /dev/null +++ b/src/Dockerfile-gpu @@ -0,0 +1,21 @@ +FROM tensorflow/tensorflow:1.12.0-gpu + +# utils +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils + +#RUN conda install gxx_linux-64 + +#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential + +# Grab requirements.txt. +COPY requirements-gpu.txt /tmp/requirements-gpu.txt + +# Install dependencies +RUN pip install -qr /tmp/requirements-gpu.txt + +# create a user for web server +RUN adduser --disabled-password --gecos "" foo + +COPY ./ /opt/src + +WORKDIR /opt/src \ No newline at end of file diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py index b253edd..f088cb3 100644 --- a/src/app/apis/SentimentV1/API_helpers_nlp.py +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -27,7 +27,7 @@ def save_classes_label_dict(label_dict, file_path_name): logging.info("* Helper: Classes Label Json Saved") -def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): +def download_a_dir_from_s3(bucket_name, local_path): """ download the folder from S3 @@ -35,7 +35,7 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): Will not download if the local folder already exists """ - logging.info("* Helper: Loading Text from S3 {} {}".format(bucket_name,bucket_prefix)) + logging.info("* Helper: Loading Text from S3 {} ".format(bucket_name)) path = os.path.join(bucket_name,'data') output_path = os.path.join(local_path, bucket_name) save_path = os.path.join(local_path, path) @@ -72,7 +72,7 @@ def download_test_file_from_s3(bucket_name, bucket_prefix, local_path): Will not download if the local folder already exists """ - logging.info("* Helper: Loading Text from S3 {} {}".format(bucket_name,bucket_prefix)) + logging.info("* Helper: Loading Text from S3 {} ".format(bucket_name)) path = os.path.join(bucket_name,'data') output_path = os.path.join(local_path, bucket_name) save_path = os.path.join(local_path, path) diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py index 1ea6c79..f01616b 100644 --- a/src/app/apis/SentimentV1/sentimentV1.py +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -84,9 +84,9 @@ def run_train_bert(): Finetune BERT uncased small language model """ s3_bucket_name = request.form.get('train_bucket_name') - model_name = request.form.get('model_name') + #model_name = request.form.get('model_name') + model_name = s3_bucket_name local_data_path = os.path.join('./tmp') - s3_bucket_prefix = '' batch_size = 32 nb_epoch = 3 @@ -96,7 +96,6 @@ def run_train_bert(): async_train_bert.apply_async((model_name, local_data_path, s3_bucket_name, - s3_bucket_prefix, nb_epoch, batch_size, this_id), task_id=this_id) @@ -111,9 +110,9 @@ def run_test_bert(): Test sentences using BERT fine tuned model """ s3_bucket_name = request.form.get('test_bucket_name') - model_name = request.form.get('model_name') + #model_name = request.form.get('model_name') + model_name = s3_bucket_name local_data_path = os.path.join('./tmp') - s3_bucket_prefix = '' batch_size = 32 nb_epoch = 3 @@ -123,7 +122,6 @@ def run_test_bert(): async_test_bert.apply_async((model_name, local_data_path, s3_bucket_name, - s3_bucket_prefix, nb_epoch, batch_size, this_id), task_id=this_id) diff --git a/src/app/index.html b/src/app/index.html new file mode 100644 index 0000000..2cb9375 --- /dev/null +++ b/src/app/index.html @@ -0,0 +1,22 @@ + + + + +//button +
+

Test

+
+ +
+ +
diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py index f476677..a1c4cd2 100644 --- a/src/app/tasks_nlp.py +++ b/src/app/tasks_nlp.py @@ -28,7 +28,6 @@ def async_train_bert(model_name, local_data_path, s3_bucket_name, - s3_bucket_prefix, nb_epoch, batch_size, id): @@ -36,8 +35,7 @@ def async_train_bert(model_name, train a model using BERT pre-trained model """ text_data_path = API_helpers_nlp.download_a_dir_from_s3(s3_bucket_name, - s3_bucket_prefix, - local_path = TEMP_FOLDER) + local_path = TEMP_FOLDER) logging.info('*Text Data Path:%s',text_data_path) try: @@ -55,7 +53,6 @@ def async_train_bert(model_name, def async_test_bert(model_name, local_data_path, s3_bucket_name, - s3_bucket_prefix, nb_epoch, batch_size, id): @@ -63,7 +60,6 @@ def async_test_bert(model_name, train a model using BERT pre-trained model """ text_data_path = API_helpers_nlp.download_test_file_from_s3(s3_bucket_name, - s3_bucket_prefix, local_path = TEMP_FOLDER) logging.info('*Text Data Path:%s',text_data_path) diff --git a/src/requirements-gpu.txt b/src/requirements-gpu.txt new file mode 100644 index 0000000..da80471 --- /dev/null +++ b/src/requirements-gpu.txt @@ -0,0 +1,98 @@ +absl-py==0.2.2 +appnope==0.1.0 +asn1crypto==0.24.0 +astor==0.6.2 +aws==0.2.5 +awscli==1.15.38 +backports-abc==0.5 +backports.functools-lru-cache==1.5 +backports.shutil-get-terminal-size==1.0.0 +backports.weakref==1.0.post1 +bcdoc==0.12.2 +bcrypt==3.1.4 +bleach==1.5.0 +boto==2.48.0 +boto3==1.7.38 +botocore==1.10.38 +certifi==2018.4.16 +celery==4.2.0 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +colorama==0.2.5 +cryptography==2.2.2 +cycler==0.10.0 +decorator==4.3.0 +Django==1.11.15 +docutils==0.14 +enum34==1.1.6 +envparse==0.2.0 +fabric==2.1.3 +Flask==1.0.2 +funcsigs==1.0.2 +futures==3.2.0 +gast==0.2.0 +grpcio==1.12.1 +h5py==2.8.0 +html5lib==0.9999999 +idna==2.7 +image==1.5.24 +invoke==1.0.0 +ipaddress==1.0.22 +ipykernel==4.8.2 +ipython==5.7.0 +ipython-genutils==0.2.0 +itsdangerous==0.24 +Jinja2==2.10 +jmespath==0.9.3 +jupyter-client==5.2.3 +jupyter-core==4.4.0 +Keras==2.0.0 +Keras-Applications==1.0.2 +Keras-Preprocessing==1.0.1 +kiwisolver==1.0.1 +Markdown==2.6.11 +MarkupSafe==1.0 +matplotlib==2.2.2 +mock==2.0.0 +numpy==1.14.4 +paramiko==2.4.2 +pathlib2==2.3.2 +pbr==4.0.4 +pexpect==4.6.0 +pickleshare==0.7.4 +Pillow==5.1.0 +prettytable==0.7.2 +prompt-toolkit==1.0.15 +protobuf==3.6.0 +ptyprocess==0.5.2 +pyasn1==0.4.3 +pycparser==2.18 +Pygments==2.2.0 +PyNaCl==1.2.1 +pyparsing==2.2.0 +python-dateutil==2.7.3 +pytz==2018.4 +PyYAML==3.12 +pyzmq==17.0.0 +redis==2.10.6 +requests==2.19.0 +rq==0.11.0 +rsa==3.1.2 +s3transfer==0.1.13 +scandir==1.7 +scipy==1.1.0 +simplegeneric==0.8.1 +singledispatch==3.4.0.3 +six==1.11.0 +subprocess32==3.5.1 +tensorboard==1.8.0 +tensorflow-gpu==1.11.0 +termcolor==1.1.0 +tornado==5.0.2 +tqdm==4.26.0 +traitlets==4.3.2 +urllib3==1.23 +uWSGI==2.0.17 +wcwidth==0.1.7 +Werkzeug==0.14.1 diff --git a/src/requirements.txt b/src/requirements.txt index da80471..015ff3f 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -87,7 +87,7 @@ singledispatch==3.4.0.3 six==1.11.0 subprocess32==3.5.1 tensorboard==1.8.0 -tensorflow-gpu==1.11.0 +tensorflow==1.11.0 termcolor==1.1.0 tornado==5.0.2 tqdm==4.26.0 diff --git a/src/requirementssenti.txt b/src/requirementssenti.txt index 95ab373..ff7f09c 100644 --- a/src/requirementssenti.txt +++ b/src/requirementssenti.txt @@ -88,7 +88,7 @@ singledispatch==3.4.0.3 six==1.11.0 subprocess32==3.5.1 tensorboard==1.8.0 -tensorflow-gpu==1.11.0 +tensorflow==1.11.0 termcolor==1.1.0 tornado==5.0.2 tqdm==4.26.0 From c71655141b2b9220057125d9c48ce7b3e85fcd74 Mon Sep 17 00:00:00 2001 From: Manu Date: Mon, 4 Feb 2019 22:35:08 -0800 Subject: [PATCH 16/18] updated readme to use docker-compose-gpu.yml for gpu training --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fd761d5..d3b747d 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Move to the directory where you cloned *Sherlock* , and run: ```bash docker-compose up --build ``` +Training using BERT runs much faster on GPU with >12GB RAM (Tested with Nvidia K80). To train with GPU run: +```bash +docker-compose -f docker-compose-gpu.yml up --build +``` If everything goes well, you should start seeing the building message of the docker containers: ``` From e7cbc96cec5f564d830f537393bb3aec793c14fb Mon Sep 17 00:00:00 2001 From: Manu Date: Tue, 5 Feb 2019 14:13:04 -0800 Subject: [PATCH 17/18] updated docker image for sentiment inference server --- src/Dockerfile-sentiment | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Dockerfile-sentiment b/src/Dockerfile-sentiment index 9932b1a..cbe3f1d 100644 --- a/src/Dockerfile-sentiment +++ b/src/Dockerfile-sentiment @@ -1,13 +1,12 @@ -FROM tensorflow/tensorflow:latest-gpu +FROM continuumio/miniconda:4.4.10 # utils RUN apt-get update && apt-get install -y --no-install-recommends apt-utils -#RUN conda install gxx_linux-64 +RUN conda install gxx_linux-64 -#RUN conda install python=3.6 -#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential +RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential # Grab requirements.txt. COPY requirementssenti.txt /tmp/requirementssenti.txt From 061177fcb3d582903d497080d9e3c3e743fb8ecc Mon Sep 17 00:00:00 2001 From: Manu Suryavansh Date: Tue, 19 Feb 2019 14:52:37 -0800 Subject: [PATCH 18/18] Update README.md link for NLP slides --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d3b747d..db0548a 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ Sherlock is currently serving as RESTful APIs. - [Sherlock for NLP](#sherlock-for-nlp) +[Here](http://bit.ly/sherlock-nlp) are the slides for Sherlock NLP + [Here](http://bit.ly/michaniki_demo) are the slides for project Sherlock (previously called Michaniki). @@ -322,4 +324,4 @@ curl -X POST \ -F test_bucket_name=YOUR_BUCKET_NAME \ -F test_bucket_prefix=YOUR_MODEL_NAME ``` -At the end of prediction a file named 'test_results.csv' will be uploaded to the same S3 bucket. \ No newline at end of file +At the end of prediction a file named 'test_results.csv' will be uploaded to the same S3 bucket.