diff --git a/.gitignore b/.gitignore
index 05e8229..641b4ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ __pycache__/
# model files
*.h5
+*.bin
# Distribution / packaging
.Python
diff --git a/README.md b/README.md
index 2e0e24e..db0548a 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,8 @@ Sherlock is currently serving as RESTful APIs.
- [Sherlock for NLP](#sherlock-for-nlp)
+[Here](http://bit.ly/sherlock-nlp) are the slides for Sherlock NLP
+
[Here](http://bit.ly/michaniki_demo) are the slides for project Sherlock (previously called Michaniki).
@@ -78,6 +80,10 @@ Move to the directory where you cloned *Sherlock* , and run:
```bash
docker-compose up --build
```
+Training using BERT runs much faster on GPU with >12GB RAM (Tested with Nvidia K80). To train with GPU run:
+```bash
+docker-compose -f docker-compose-gpu.yml up --build
+```
If everything goes well, you should start seeing the building message of the docker containers:
```
@@ -271,20 +277,20 @@ curl -X POST \
### 2. Train a new classification model using pre-trained BERT model
-**The new text dataset should be stored at S3 first, with the directory architecture in S3 should look like this**:
+**The new text dataset should be stored at S3 first, the directory architecture in S3 should look like this**:
```
.
├── YOUR_BUCKET_NAME
│ ├── train.tsv
-│ ├── dev.tsv
+│ ├── val.tsv
│ ├── test.tsv
```
The folder name you give to *YOUR_MODEL_NAME* will be used to identify this model once it get trained.
-The name of train, dev and test files **can't be changed**.
+The name of train, val and test files **can't be changed**.
The train and dev file should have below format (without header)-
id label None Sentence
-1 0 NC
+1 0 NC Text
The test.tsv file should only have id and sentence column (with header)
**The S3 folders should have public access permission**.
@@ -297,4 +303,25 @@ curl -X POST \
-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
-F train_bucket_name=YOUR_BUCKET_NAME \
-F train_bucket_prefix=YOUR_MODEL_NAME
-```
\ No newline at end of file
+```
+### 3. Lable all text in a csv file using pre-trained BERT model
+
+**The new test tsv file should be stored at the same S3 bucket as above for that model, directory architecture in S3 should look like this**:
+```
+.
+├── YOUR_BUCKET_NAME
+│ ├── train.tsv
+│ ├── val.tsv
+│ ├── test.tsv
+```
+To call this API do:
+```bash
+curl -X POST \
+ http://127.0.0.1:3031/sentimentV1/testbert \
+ -H 'Cache-Control: no-cache' \
+ -H 'Postman-Token: 4e90e1d6-de18-4501-a82c-f8a878616b12' \
+ -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
+ -F test_bucket_name=YOUR_BUCKET_NAME \
+ -F test_bucket_prefix=YOUR_MODEL_NAME
+```
+At the end of prediction a file named 'test_results.csv' will be uploaded to the same S3 bucket.
diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml
new file mode 100644
index 0000000..bab364c
--- /dev/null
+++ b/docker-compose-gpu.yml
@@ -0,0 +1,99 @@
+version: '2.3'
+
+services:
+ michaniki_client:
+ build:
+ context: ./src
+ dockerfile: Dockerfile-gpu
+ runtime: nvidia
+ ports:
+ - "3031:3031"
+ environment:
+ - PORT=3031
+ - FLAS_APP=app/__init__.py
+ - FLASK_DEBUG=1
+ - REDIS_URL="redis://redis"
+ - REDIS_PORT=6379
+ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+
+ volumes:
+ - ./src:/opt/src
+
+ command: ./entryPoint.sh
+ depends_on:
+ - redis
+ networks:
+ - michaniki
+
+ inference_server:
+ build:
+ context: ./src
+ dockerfile: Dockerfile-gpu
+ runtime: nvidia
+ environment:
+ - REDIS_URL="redis://redis"
+ - REDIS_PORT=6379
+ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+
+ command: ['python', 'app/models/InceptionV3/inception_inference_server.py']
+ volumes:
+ - ./src:/opt/src
+
+ networks:
+ - michaniki
+ depends_on:
+ - michaniki_client
+ - redis
+
+ sentiment_inference_server:
+ build:
+ context: ./src
+ dockerfile: Dockerfile-sentiment
+ runtime: nvidia
+ environment:
+ - REDIS_URL="redis://redis"
+ - REDIS_PORT=6379
+ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+ command: ['python', 'app/models/SentimentV1/sentiment_infer_server.py']
+ volumes:
+ - ./src:/opt/src
+ networks:
+ - michaniki
+ depends_on:
+ - michaniki_client
+ - redis
+
+ celery_worker:
+ build:
+ context: ./src
+ dockerfile: Dockerfile-gpu
+ runtime: nvidia
+ command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info']
+ volumes:
+ - ./src:/opt/src
+ networks:
+ - michaniki
+ depends_on:
+ - michaniki_client
+ environment:
+ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+ - DB_HOST=db
+ - DB_USERNAME=root
+ - DB_PASSWORD=michaniki
+ - DB_NAME=michanikidb
+ - BROKER_URL=redis://redis:6379/0
+
+
+ redis:
+ image: redis:4.0.5-alpine
+ command: ["redis-server", "--appendonly", "yes"]
+ hostname: redis
+ networks:
+ - michaniki
+
+networks:
+ michaniki:
diff --git a/docker-compose.yml b/docker-compose.yml
index 6561d3c..a7d1b45 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,7 +3,6 @@ version: '2.3'
services:
michaniki_client:
build: ./src
- runtime: nvidia
ports:
- "3031:3031"
environment:
@@ -26,7 +25,6 @@ services:
inference_server:
build: ./src
- runtime: nvidia
environment:
- REDIS_URL="redis://redis"
- REDIS_PORT=6379
@@ -47,7 +45,6 @@ services:
build:
context: ./src
dockerfile: Dockerfile-sentiment
- runtime: nvidia
environment:
- REDIS_URL="redis://redis"
- REDIS_PORT=6379
@@ -64,7 +61,6 @@ services:
celery_worker:
build: ./src
- runtime: nvidia
command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info']
volumes:
- ./src:/opt/src
diff --git a/src/Dockerfile b/src/Dockerfile
index 577b4d2..3c8e96e 100644
--- a/src/Dockerfile
+++ b/src/Dockerfile
@@ -1,11 +1,11 @@
-FROM tensorflow/tensorflow:latest-gpu
+FROM continuumio/miniconda:4.4.10
# utils
RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
-#RUN conda install gxx_linux-64
+RUN conda install gxx_linux-64
-#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential
+RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential
# Grab requirements.txt.
COPY requirements.txt /tmp/requirements.txt
diff --git a/src/Dockerfile-gpu b/src/Dockerfile-gpu
new file mode 100644
index 0000000..e7dcecb
--- /dev/null
+++ b/src/Dockerfile-gpu
@@ -0,0 +1,21 @@
+FROM tensorflow/tensorflow:1.12.0-gpu
+
+# utils
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
+
+#RUN conda install gxx_linux-64
+
+#RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential
+
+# Grab requirements.txt.
+COPY requirements-gpu.txt /tmp/requirements-gpu.txt
+
+# Install dependencies
+RUN pip install -qr /tmp/requirements-gpu.txt
+
+# create a user for web server
+RUN adduser --disabled-password --gecos "" foo
+
+COPY ./ /opt/src
+
+WORKDIR /opt/src
\ No newline at end of file
diff --git a/src/Dockerfile-sentiment b/src/Dockerfile-sentiment
index 5b5c09a..cbe3f1d 100644
--- a/src/Dockerfile-sentiment
+++ b/src/Dockerfile-sentiment
@@ -1,11 +1,10 @@
-FROM continuumio/miniconda3:4.5.12
+FROM continuumio/miniconda:4.4.10
# utils
RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
RUN conda install gxx_linux-64
-RUN conda install python=3.6
RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential
@@ -15,6 +14,8 @@ COPY requirementssenti.txt /tmp/requirementssenti.txt
# Install dependencies
RUN pip install -qr /tmp/requirementssenti.txt
+RUN pip install fasttext
+
# create a user for web server
RUN adduser --disabled-password --gecos "" foo
diff --git a/src/app/apis/InceptionV3/API_helpers.py b/src/app/apis/InceptionV3/API_helpers.py
index 2beef9b..180fe86 100644
--- a/src/app/apis/InceptionV3/API_helpers.py
+++ b/src/app/apis/InceptionV3/API_helpers.py
@@ -69,7 +69,10 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path):
os.makedirs(save_path)
except OSError:
pass
- mybucket.download_file(obj.key, os.path.join(save_path, filename))
+ try:
+ mybucket.download_file(obj.key, os.path.join(save_path, filename))
+ except OSError:
+ pass
print "* Helper: Images Loaded at: {}".format(output_path)
return output_path
diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py
index 72bf292..f088cb3 100644
--- a/src/app/apis/SentimentV1/API_helpers_nlp.py
+++ b/src/app/apis/SentimentV1/API_helpers_nlp.py
@@ -27,7 +27,7 @@ def save_classes_label_dict(label_dict, file_path_name):
logging.info("* Helper: Classes Label Json Saved")
-def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path):
+def download_a_dir_from_s3(bucket_name, local_path):
"""
download the folder from S3
@@ -35,14 +35,14 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path):
Will not download if the local folder already exists
"""
- logging.info("* Helper: Loading Text from S3 {} {}".format(bucket_name,bucket_prefix))
+ logging.info("* Helper: Loading Text from S3 {} ".format(bucket_name))
path = os.path.join(bucket_name,'data')
output_path = os.path.join(local_path, bucket_name)
save_path = os.path.join(local_path, path)
logging.info('*Saving text files at:%s',save_path)
s3 = boto3.resource('s3')
- mybucket = s3.Bucket(bucket_name)
+
# if blank prefix is given, return everything)
key1 = 'train.tsv'
key2 = 'val.tsv'
@@ -54,6 +54,41 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path):
try:
s3.Bucket(bucket_name).download_file(key1, os.path.join(save_path,'train.tsv'))
s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'dev.tsv'))
+
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == "404":
+ print("The object does not exist.")
+ else:
+ raise
+
+ logging.info("* Helper: Text Loaded at: {}".format(output_path))
+ return output_path
+
+def download_test_file_from_s3(bucket_name, bucket_prefix, local_path):
+ """
+ download the folder from S3
+
+ local: /src/tmp/model_data/
+
+ Will not download if the local folder already exists
+ """
+ logging.info("* Helper: Loading Text from S3 {} ".format(bucket_name))
+ path = os.path.join(bucket_name,'data')
+ output_path = os.path.join(local_path, bucket_name)
+ save_path = os.path.join(local_path, path)
+ logging.info('*Saving text files at:%s',save_path)
+
+ s3 = boto3.resource('s3')
+ mybucket = s3.Bucket(bucket_name)
+ # if blank prefix is given, return everything)
+
+ key3 = 'test.tsv'
+ try:
+ os.makedirs(save_path)
+ except OSError:
+ pass
+ try:
+ mybucket.download_file(key3, os.path.join(save_path,'test.tsv'))
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
print("The object does not exist.")
diff --git a/src/app/apis/SentimentV1/__init__.py b/src/app/apis/SentimentV1/__init__.py
index 1d6da5b..07b685a 100644
--- a/src/app/apis/SentimentV1/__init__.py
+++ b/src/app/apis/SentimentV1/__init__.py
@@ -4,3 +4,4 @@
@author: manu
'''
from sentimentV1 import *
+from API_helpers_nlp import *
\ No newline at end of file
diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py
index 8d1b079..f01616b 100644
--- a/src/app/apis/SentimentV1/sentimentV1.py
+++ b/src/app/apis/SentimentV1/sentimentV1.py
@@ -24,7 +24,10 @@
# michaniki app
from ...tasks_nlp import async_train_bert
+from ...tasks_nlp import async_test_bert
+SENTIMENT_TEXT_QUEUE = app.config['SENTIMENT_TEXT_QUEUE']
+CLIENT_SLEEP = app.config['CLIENT_SLEEP']
# temp folder save image files downloaded from S3
TEMP_FOLDER = os.path.join('./tmp')
@@ -81,9 +84,9 @@ def run_train_bert():
Finetune BERT uncased small language model
"""
s3_bucket_name = request.form.get('train_bucket_name')
- model_name = request.form.get('model_name')
+ #model_name = request.form.get('model_name')
+ model_name = s3_bucket_name
local_data_path = os.path.join('./tmp')
- s3_bucket_prefix = ''
batch_size = 32
nb_epoch = 3
@@ -93,7 +96,6 @@ def run_train_bert():
async_train_bert.apply_async((model_name,
local_data_path,
s3_bucket_name,
- s3_bucket_prefix,
nb_epoch,
batch_size,
this_id), task_id=this_id)
@@ -101,3 +103,29 @@ def run_train_bert():
"task_id": this_id,
"status": "Retraining and Fine-Tuning usign BERT is Initiated"
}), 200
+
+@blueprint.route('/testbert', methods=['POST'])
+def run_test_bert():
+ """
+ Test sentences using BERT fine tuned model
+ """
+ s3_bucket_name = request.form.get('test_bucket_name')
+ #model_name = request.form.get('model_name')
+ model_name = s3_bucket_name
+ local_data_path = os.path.join('./tmp')
+ batch_size = 32
+ nb_epoch = 3
+
+ # create a celer task id
+ this_id = celery.uuid()
+
+ async_test_bert.apply_async((model_name,
+ local_data_path,
+ s3_bucket_name,
+ nb_epoch,
+ batch_size,
+ this_id), task_id=this_id)
+ return jsonify({
+ "task_id": this_id,
+ "status": "Testing usign BERT is Started. Results uploaded to S3 bucket"
+ }), 200
\ No newline at end of file
diff --git a/src/app/index.html b/src/app/index.html
new file mode 100644
index 0000000..2cb9375
--- /dev/null
+++ b/src/app/index.html
@@ -0,0 +1,22 @@
+
+
+
+
+//button
+
diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py
index 6bf6965..4bdbc96 100644
--- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py
+++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py
@@ -1,11 +1,13 @@
'''
Created on Jan 13, 2018
-@author: manu
+Most of the code is taken from run_classifier.py from
+the Google BERT github - https://github.com/google-research/bert
'''
import os
import glob
import logging
+import boto3
#Import BERT related file - USED as is from Google/BERT
import modeling
@@ -162,7 +164,138 @@ def traineval_model(self, local_dir,
for key in sorted(result.keys()):
tf.logging.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
-
+
return result
+
+ def test_model(self, local_dir,
+ nb_epoch,
+ batch_size,
+ bucket_name):
+ """
+ Use the BERT Uncased language model to train on
+ new data
+ """
+ tf.logging.set_verbosity(tf.logging.INFO)
+ logging.info("*:BERT MODEL PATH:%s",BERT_MODEL_PATH)
+ logging.info("*:Local Dir%s",local_dir)
+
+
+ mod_name = self.model_name
+ BERT_MODEL = 'uncased_L-12_H-768_A-12'
+ BERT_PRETRAINED_DIR = BERT_MODEL_PATH
+ OUTPUT_DIR = os.path.join(local_dir,'output_bert')
+ DATA_DIR = os.path.join(local_dir,'data')
+ logging.info('***** Model output directory: %s*****',OUTPUT_DIR)
+ logging.info('***** BERT pretrained directory: %s *****',BERT_PRETRAINED_DIR)
+ logging.info('***** DATA directory: %s *****',DATA_DIR)
+ TRAIN_BATCH_SIZE = 32
+ EVAL_BATCH_SIZE = 8
+ PREDICT_BATCH_SIZE = 32
+ LEARNING_RATE = 2e-5
+ NUM_TRAIN_EPOCHS = 3.0
+ WARMUP_PROPORTION = 0.1
+ MAX_SEQ_LENGTH = 128
+ # Model configs
+ # if you wish to finetune a model on a larger dataset, use larger interval
+ SAVE_CHECKPOINTS_STEPS = 1000
+ # each checpoint weights about 1,5gb
+ ITERATIONS_PER_LOOP = 1000
+ NUM_TPU_CORES = 8
+
+ VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR,'vocab.txt')
+ BERT_CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR,'bert_config.json')
+ with open(os.path.join(OUTPUT_DIR,'final_ckpt.txt')) as f:
+ content = f.readlines()
+ logging.info("***Final_cktp->%s\n",content)
+ test_ckpt = content[0].split('/')[-1]
+ INIT_CHECKPOINT = os.path.join(OUTPUT_DIR, test_ckpt)
+ DO_LOWER_CASE = BERT_MODEL.startswith('uncased')
+
+ logging.info("Found VOCAB File:%s",VOCAB_FILE)
+ bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE)
+ tf.gfile.MakeDirs(OUTPUT_DIR)
+ processor = run_classifier.ColaProcessor()
+ label_list = processor.get_labels()
+ tokenizer = tokenization.FullTokenizer(
+ vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)
+
+ # Since training will happen on GPU, we won't need a cluster resolver
+ tpu_cluster_resolver = None
+ # TPUEstimator also supports training on CPU and GPU. You don't need to define a separate tf.estimator.Estimator.
+ run_config = tf.contrib.tpu.RunConfig(
+ cluster=tpu_cluster_resolver,
+ model_dir=OUTPUT_DIR,
+ save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,
+ tpu_config=tf.contrib.tpu.TPUConfig(
+ iterations_per_loop=ITERATIONS_PER_LOOP,
+ num_shards=NUM_TPU_CORES,
+ per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))
+
+ train_examples = None
+ num_train_steps = None
+ num_warmup_steps = None
+
+
+ model_fn = run_classifier.model_fn_builder(
+ bert_config=bert_config,
+ num_labels=len(label_list),
+ init_checkpoint=INIT_CHECKPOINT,
+ learning_rate=LEARNING_RATE,
+ num_train_steps=num_train_steps,
+ num_warmup_steps=num_warmup_steps,
+ use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available
+ use_one_hot_embeddings=False) #Try with True
+
+ estimator = tf.contrib.tpu.TPUEstimator(
+ use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available
+ model_fn=model_fn,
+ config=run_config,
+ train_batch_size=TRAIN_BATCH_SIZE,
+ eval_batch_size=EVAL_BATCH_SIZE,
+ predict_batch_size=PREDICT_BATCH_SIZE)
+
+ predict_examples = processor.get_test_examples(DATA_DIR)
+ num_actual_predict_examples = len(predict_examples)
+ predict_file = os.path.join(OUTPUT_DIR, "predict.tf_record")
+ run_classifier.file_based_convert_examples_to_features(predict_examples, label_list,
+ MAX_SEQ_LENGTH, tokenizer,
+ predict_file)
+
+ tf.logging.info("***** Running prediction*****")
+ tf.logging.info(" Num examples = %d (%d actual, %d padding)",
+ len(predict_examples), num_actual_predict_examples,
+ len(predict_examples) - num_actual_predict_examples)
+ tf.logging.info(" Batch size = %d", batch_size)
+
+ predict_input_fn = run_classifier.file_based_input_fn_builder(
+ input_file=predict_file,
+ seq_length=MAX_SEQ_LENGTH,
+ is_training=False,
+ drop_remainder=False)
+
+ result = estimator.predict(input_fn=predict_input_fn)
+ output_predict_file = os.path.join(OUTPUT_DIR, "test_results.tsv")
+ with tf.gfile.GFile(output_predict_file, "w") as writer:
+ num_written_lines = 0
+ tf.logging.info("***** Predict results *****")
+ for (i, prediction) in enumerate(result):
+ probabilities = prediction["probabilities"]
+ if i >= num_actual_predict_examples:
+ break
+ output_line = "\t".join(
+ str(class_probability)
+ for class_probability in probabilities) + "\n"
+ writer.write(output_line)
+ num_written_lines += 1
+ assert num_written_lines == num_actual_predict_examples
+ s3 = boto3.resource('s3')
+ tf.logging.info("Done with prediction uploading results to S3")
+ try:
+ s3.Bucket(bucket_name).upload_file(output_predict_file, output_predict_file)
+ except Exception as err:
+ logging.info("Unable to upload to S3")
+ logging.info(err)
+
+ return 1
diff --git a/src/app/models/SentimentV1/sentiment_infer_server.py b/src/app/models/SentimentV1/sentiment_infer_server.py
index e2cb3bd..e786cf7 100644
--- a/src/app/models/SentimentV1/sentiment_infer_server.py
+++ b/src/app/models/SentimentV1/sentiment_infer_server.py
@@ -7,10 +7,10 @@
import redis
import time
import json
-from textblob import TextBlob
import logging
from collections import defaultdict
-
+import fasttext
+import re
#helpers
import settings
@@ -22,6 +22,11 @@ class sentimentV1_inference_server:
def __init__(self):
# pre-load some models here on start
self.loaded_models = {}
+
+ def strip_formatting(self, string):
+ string = string.lower()
+ string = re.sub(r"([.!?,'/()])", r" \1 ", string)
+ return string
def run_sentimentV1_infernece_server(self):
'''
@@ -33,6 +38,9 @@ def run_sentimentV1_infernece_server(self):
Sentecnes are tracked using is their id
'''
logging.info("Sentiment Inference Server running")
+ FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin")
+ logging.info("IMDB Model path:%s",FAST_IMDB_MODEL_PATH)
+
while True:
queue = db.lrange(settings.TEXT_QUEUE, 0, settings.BATCH_SIZE) #Is this queue different from the Queue in API path
textIDs = defaultdict(list) #dict to hold sentence and id for a model type
@@ -55,14 +63,20 @@ def run_sentimentV1_infernece_server(self):
if textIDs:
+ classifier = fasttext.load_model(FAST_IMDB_MODEL_PATH)
logging.info("* Predicting for {} of Models".format(len(textIDs.keys())))
logging.info("* Number of Sentences: {}".format(num_text))
-
+ reviews=[]
for t in text_list:
logging.info("Text is:%s",t["text"])
- preds = TextBlob(t["text"])
- res = {"polarity":preds.sentiment.polarity,"subjectvity":preds.sentiment.subjectivity}
+ reviews.append(t["text"])
+ preprocessed_reviews = list(map(self.strip_formatting, reviews))
+ result = classifier.predict_proba(preprocessed_reviews, 1)
+ if result[0][0][0] == '__label__1':
+ label = 'positive'
+ else: label = 'negative'
+ res = {"label":label,"probability":result[0][0][1]}
db.set(t["id"], json.dumps(res))
db.ltrim(settings.TEXT_QUEUE, len(textIDs), -1)
diff --git a/src/app/tasks_nlp.py b/src/app/tasks_nlp.py
index 3c58a59..a1c4cd2 100644
--- a/src/app/tasks_nlp.py
+++ b/src/app/tasks_nlp.py
@@ -28,7 +28,6 @@
def async_train_bert(model_name,
local_data_path,
s3_bucket_name,
- s3_bucket_prefix,
nb_epoch,
batch_size,
id):
@@ -36,8 +35,7 @@ def async_train_bert(model_name,
train a model using BERT pre-trained model
"""
text_data_path = API_helpers_nlp.download_a_dir_from_s3(s3_bucket_name,
- s3_bucket_prefix,
- local_path = TEMP_FOLDER)
+ local_path = TEMP_FOLDER)
logging.info('*Text Data Path:%s',text_data_path)
try:
@@ -50,3 +48,28 @@ def async_train_bert(model_name,
logging.info(err)
#shutil.rmtree(text_data_path, ignore_errors=True)
raise
+
+@michaniki_celery_app.task()
+def async_test_bert(model_name,
+ local_data_path,
+ s3_bucket_name,
+ nb_epoch,
+ batch_size,
+ id):
+ """
+ train a model using BERT pre-trained model
+ """
+ text_data_path = API_helpers_nlp.download_test_file_from_s3(s3_bucket_name,
+ local_path = TEMP_FOLDER)
+
+ logging.info('*Text Data Path:%s',text_data_path)
+ try:
+ bert_transfer = sentimentV1_transfer_retraining.BertTransferLeaner(model_name)
+ new_model_eval_res = bert_transfer.test_model(text_data_path,nb_epoch,batch_size,s3_bucket_name)
+ logging.info("****Test done, file saved in S3")
+ print(new_model_eval_res)
+ return str(1),str(1)
+ except Exception as err:
+ logging.info(err)
+ #shutil.rmtree(text_data_path, ignore_errors=True)
+ raise
diff --git a/src/install_base_model.py b/src/install_base_model.py
index 031c6bb..a74f5bd 100644
--- a/src/install_base_model.py
+++ b/src/install_base_model.py
@@ -5,12 +5,16 @@
import zipfile, io
from tqdm import tqdm
import math
+import boto3
+import botocore
from keras.applications.inception_v3 import InceptionV3
BASE_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "base", "base.h5")
TOPLESS_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "topless")
-BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt.data-00000-of-00001")
+BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt")
BERT_DIR_PATH = os.path.join("app", "models", "SentimentV1")
+FAST_IMDB_DIR = os.path.join("app", "models", "SentimentV1","fastimdb")
+FAST_IMDB_MODEL_PATH = os.path.join("app", "models", "SentimentV1","fastimdb","imdb_model.bin")
# loading base model
if os.path.exists(BASE_MODEL_PATH):
@@ -48,6 +52,23 @@
z = zipfile.ZipFile(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip'))
z.extractall(BERT_DIR_PATH)
+ os.remove(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip'))
+
+if os.path.exists(FAST_IMDB_MODEL_PATH):
+ print "* Found Base IMDB model"
+else:
+ logging.info("Base IMDB model not found. Downloading....")
+ s3 = boto3.resource('s3')
+ mybucket = s3.Bucket('fastimdb')
+ model_name = 'imdb_model.bin'
+ try:
+ os.makedirs(FAST_IMDB_DIR)
+ except OSError:
+ pass
+ try:
+ mybucket.download_file(model_name, os.path.join(FAST_IMDB_DIR,'imdb_model.bin'))
+ except Exception as err:
+ logging.info(err)
# clean up the died images upon start:
# need to wait a bit for redis container to start up
diff --git a/src/requirements-gpu.txt b/src/requirements-gpu.txt
new file mode 100644
index 0000000..da80471
--- /dev/null
+++ b/src/requirements-gpu.txt
@@ -0,0 +1,98 @@
+absl-py==0.2.2
+appnope==0.1.0
+asn1crypto==0.24.0
+astor==0.6.2
+aws==0.2.5
+awscli==1.15.38
+backports-abc==0.5
+backports.functools-lru-cache==1.5
+backports.shutil-get-terminal-size==1.0.0
+backports.weakref==1.0.post1
+bcdoc==0.12.2
+bcrypt==3.1.4
+bleach==1.5.0
+boto==2.48.0
+boto3==1.7.38
+botocore==1.10.38
+certifi==2018.4.16
+celery==4.2.0
+cffi==1.11.5
+chardet==3.0.4
+click==6.7
+colorama==0.2.5
+cryptography==2.2.2
+cycler==0.10.0
+decorator==4.3.0
+Django==1.11.15
+docutils==0.14
+enum34==1.1.6
+envparse==0.2.0
+fabric==2.1.3
+Flask==1.0.2
+funcsigs==1.0.2
+futures==3.2.0
+gast==0.2.0
+grpcio==1.12.1
+h5py==2.8.0
+html5lib==0.9999999
+idna==2.7
+image==1.5.24
+invoke==1.0.0
+ipaddress==1.0.22
+ipykernel==4.8.2
+ipython==5.7.0
+ipython-genutils==0.2.0
+itsdangerous==0.24
+Jinja2==2.10
+jmespath==0.9.3
+jupyter-client==5.2.3
+jupyter-core==4.4.0
+Keras==2.0.0
+Keras-Applications==1.0.2
+Keras-Preprocessing==1.0.1
+kiwisolver==1.0.1
+Markdown==2.6.11
+MarkupSafe==1.0
+matplotlib==2.2.2
+mock==2.0.0
+numpy==1.14.4
+paramiko==2.4.2
+pathlib2==2.3.2
+pbr==4.0.4
+pexpect==4.6.0
+pickleshare==0.7.4
+Pillow==5.1.0
+prettytable==0.7.2
+prompt-toolkit==1.0.15
+protobuf==3.6.0
+ptyprocess==0.5.2
+pyasn1==0.4.3
+pycparser==2.18
+Pygments==2.2.0
+PyNaCl==1.2.1
+pyparsing==2.2.0
+python-dateutil==2.7.3
+pytz==2018.4
+PyYAML==3.12
+pyzmq==17.0.0
+redis==2.10.6
+requests==2.19.0
+rq==0.11.0
+rsa==3.1.2
+s3transfer==0.1.13
+scandir==1.7
+scipy==1.1.0
+simplegeneric==0.8.1
+singledispatch==3.4.0.3
+six==1.11.0
+subprocess32==3.5.1
+tensorboard==1.8.0
+tensorflow-gpu==1.11.0
+termcolor==1.1.0
+tornado==5.0.2
+tqdm==4.26.0
+traitlets==4.3.2
+urllib3==1.23
+uWSGI==2.0.17
+wcwidth==0.1.7
+Werkzeug==0.14.1
diff --git a/src/requirements.txt b/src/requirements.txt
index da80471..015ff3f 100644
--- a/src/requirements.txt
+++ b/src/requirements.txt
@@ -87,7 +87,7 @@ singledispatch==3.4.0.3
six==1.11.0
subprocess32==3.5.1
tensorboard==1.8.0
-tensorflow-gpu==1.11.0
+tensorflow==1.11.0
termcolor==1.1.0
tornado==5.0.2
tqdm==4.26.0
diff --git a/src/requirementssenti.txt b/src/requirementssenti.txt
index a889dbe..ff7f09c 100644
--- a/src/requirementssenti.txt
+++ b/src/requirementssenti.txt
@@ -1,154 +1,99 @@
-absl-py==0.6.1
-argh==0.26.2
+absl-py==0.2.2
+appnope==0.1.0
asn1crypto==0.24.0
-astor==0.7.1
-backcall==0.1.0
+astor==0.6.2
+aws==0.2.5
+awscli==1.15.38
backports-abc==0.5
-base58==1.0.3
-beautifulsoup4==4.6.3
-bleach==3.0.2
-boto==2.49.0
-boto3==1.9.78
-botocore==1.12.78
-Bottleneck==1.2.1
-bz2file==0.98
-certifi==2018.11.29
+backports.functools-lru-cache==1.5
+backports.shutil-get-terminal-size==1.0.0
+backports.weakref==1.0.post1
+bcdoc==0.12.2
+bcrypt==3.1.4
+bleach==1.5.0
+boto==2.48.0
+boto3==1.7.38
+botocore==1.10.38
+certifi==2018.4.16
+celery==4.2.0
cffi==1.11.5
chardet==3.0.4
-Click==7.0
-cryptography==2.4.2
+click==6.7
+colorama==0.2.5
+cryptography==2.2.2
cycler==0.10.0
-cymem==2.0.2
-cytoolz==0.9.0.1
-dataclasses==0.6
+cython==0.29.4
decorator==4.3.0
-Deprecated==1.2.4
-dill==0.2.8.2
+Django==1.11.15
docutils==0.14
-entrypoints==0.2.3
-enum-compat==0.0.2
-fastai==1.0.40
-fastprogress==0.1.18
-flair==0.4.0
+enum34==1.1.6
+envparse==0.2.0
+fabric==2.1.3
Flask==1.0.2
-future==0.17.1
-future-fstrings==0.4.5
-futures==3.1.1
-gast==0.2.2
-gensim==3.4.0
-gluonnlp==0.5.0.post0
-graphviz==0.8.4
-grpcio==1.17.1
-h5py==2.9.0
-hyperopt==0.1.1
-idna==2.8
-ipykernel==5.1.0
-ipython==7.2.0
+funcsigs==1.0.2
+futures==3.2.0
+gast==0.2.0
+grpcio==1.12.1
+h5py==2.8.0
+html5lib==0.9999999
+idna==2.7
+image==1.5.24
+invoke==1.0.0
+ipaddress==1.0.22
+ipykernel==4.8.2
+ipython==5.7.0
ipython-genutils==0.2.0
-ipywidgets==7.4.2
-itsdangerous==1.1.0
-jedi==0.13.2
+itsdangerous==0.24
Jinja2==2.10
jmespath==0.9.3
-jsonschema==2.6.0
-jupyter==1.0.0
-jupyter-client==5.2.4
-jupyter-console==6.0.0
+jupyter-client==5.2.3
jupyter-core==4.4.0
-Keras==2.2.4
-Keras-Applications==1.0.6
-Keras-Preprocessing==1.0.5
+Keras==2.0.0
+Keras-Applications==1.0.2
+Keras-Preprocessing==1.0.1
kiwisolver==1.0.1
-Markdown==3.0.1
-MarkupSafe==1.1.0
-matplotlib==3.0.0
-mistune==0.8.4
-mkl-fft==1.0.6
-mkl-random==1.0.1
+Markdown==2.6.11
+MarkupSafe==1.0
+matplotlib==2.2.2
mock==2.0.0
-mpld3==0.3
-msgpack==0.5.6
-msgpack-numpy==0.4.3.2
-murmurhash==1.0.1
-mxnet-cu90==1.3.1
-mypy==0.650
-mypy-extensions==0.4.1
-networkx==2.2
-nltk==3.4
-notebook==5.7.4
-numexpr==2.6.8
-numpy==1.14.6
-olefile==0.46
-packaging==18.0
-pandas==0.23.4
-pandocfilters==1.4.2
-parso==0.3.1
-pathtools==0.1.2
-pbr==5.1.1
+numpy==1.14.4
+paramiko==2.4.2
+pathlib2==2.3.2
+pbr==4.0.4
pexpect==4.6.0
-pickleshare==0.7.5
-Pillow==5.4.1
-plac==0.9.6
-preshed==2.0.1
-prometheus-client==0.5.0
-prompt-toolkit==2.0.7
-protobuf==3.6.1
-psutil==5.4.8
-ptyprocess==0.6.0
-pycparser==2.19
-Pygments==2.3.1
-pymongo==3.7.2
-pyOpenSSL==18.0.0
-pyparsing==2.3.0
-PySocks==1.6.8
-python-dateutil==2.7.5
-pytorch-pretrained-bert==0.3.0
-pytz==2018.7
-PyYAML==3.13
-pyzmq==17.1.2
-qtconsole==4.4.3
-redis==3.0.1
-regex==2018.1.10
-requests==2.21.0
+pickleshare==0.7.4
+Pillow==5.1.0
+prettytable==0.7.2
+prompt-toolkit==1.0.15
+protobuf==3.6.0
+ptyprocess==0.5.2
+pyasn1==0.4.3
+pycparser==2.18
+Pygments==2.2.0
+PyNaCl==1.2.1
+pyparsing==2.2.0
+python-dateutil==2.7.3
+pytz==2018.4
+PyYAML==3.12
+pyzmq==17.0.0
+redis==2.10.6
+requests==2.19.0
+rq==0.11.0
+rsa==3.1.2
s3transfer==0.1.13
-scikit-learn==0.20.1
+scandir==1.7
scipy==1.1.0
-seaborn==0.9.0
-segtok==1.5.7
-Send2Trash==1.5.0
+simplegeneric==0.8.1
singledispatch==3.4.0.3
-six==1.12.0
-sklearn==0.0
-smart-open==1.8.0
-spacy==2.0.18
-sqlitedict==1.6.0
-streamlit==0.23.0
-tensorboard==1.12.2
-tensorflow==1.12.0
-tensorflow-hub==0.2.0
+six==1.11.0
+subprocess32==3.5.1
+tensorboard==1.8.0
+tensorflow==1.11.0
termcolor==1.1.0
-terminado==0.8.1
-testpath==0.4.2
-textblob==0.15.2
-thinc==6.12.1
-tokenize-rt==2.1.0
-toml==0.10.0
-toolz==0.9.0
-torch==1.0.0
-torchtext==0.3.1
-torchvision==0.2.1
-tornado==5.1.1
+tornado==5.0.2
tqdm==4.26.0
traitlets==4.3.2
-typed-ast==1.1.1
-typing==3.6.4
-tzlocal==1.5.1
-ujson==1.35
-urllib3==1.24.1
-watchdog==0.9.0
+urllib3==1.23
+uWSGI==2.0.17
wcwidth==0.1.7
-webencodings==0.5.1
Werkzeug==0.14.1
-widgetsnbextension==3.4.2
-wrapt==1.10.11
-xgboost==0.81