diff --git a/.gitignore b/.gitignore index ef66c04..ee9617b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ results/ .vscode/ +src/app/models/SentimentV1/uncased_L-12_H-768_A-12/ + +*.zip + # Byte-compiled / optimized / DLL files .DS_store __pycache__/ diff --git a/README.md b/README.md index 6c3e492..427e313 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Sherlock Sherlock is a web platform that allows user to create a image classifier for custom images, based on pre-trained CNN models. It also allows to use the customized CNN to pre-label images, and re-train the customized CNN when more training data become avaliable. -Sherlock is currently serveing as RESTful APIs. +Sherlock is currently serving as RESTful APIs. + +- [Sherlock for NLP](#sherlock-for-nlp) [Here](http://bit.ly/michaniki_demo) are the slides for project Sherlock (previously called Michaniki). @@ -253,3 +255,45 @@ curl -X POST \ -F train_bucket_prefix=S3_BUCEKT_PREFIX/sha ``` +## Sherlock for NLP + +### 1. Predict Sentiment of Text with Simple run_classifier +```bash +curl -X POST \ + http://127.0.0.1:3031/sentimentV1/predict \ + -H 'Cache-Control: no-cache' \ + -H 'Postman-Token: eeedb319-2218-44b9-86eb-63a3a1f62e14' \ + -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \ + -F textv='the movie was bad' \ + -F model_name=base +``` + +### 2. Train a new classification model using pre-trained BERT model + +**The new text dataset should be stored at S3 first, with the directory architecture in S3 should look like this**: +``` +. +├── YOUR_BUCKET_NAME +│ ├── train.tsv +│ ├── dev.tsv +│ ├── test.tsv +``` +The folder name you give to *YOUR_MODEL_NAME* will be used to identify this model once it get trained. + +The name of train, dev and test files **can't be changed**. +The train and dev file should have below format (without header)- +id label None Sentence +1 0 NC +The test.tsv file should only have id and sentence column (with header) +**The S3 folders should have public access permission**. + +To call this API, do: +```bash +curl -X POST \ + http://127.0.0.1:3031/sentimentV1/trainbert \ + -H 'Cache-Control: no-cache' \ + -H 'Postman-Token: 4e90e1d6-de18-4501-a82c-f8a878616b12' \ + -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \ + -F train_bucket_name=YOUR_BUCKET_NAME \ + -F train_bucket_prefix=YOUR_MODEL_NAME +``` diff --git a/docker-compose.yml b/docker-compose.yml index 80eb75e..3aa1857 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} volumes: - ./src:/opt/src + command: ./entryPoint.sh depends_on: - redis @@ -29,6 +30,25 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} command: ['python', 'app/models/InceptionV3/inception_inference_server.py'] + volumes: + - ./src:/opt/src + + networks: + - michaniki + depends_on: + - michaniki_client + - redis + + sentiment_inference_server: + build: + context: ./src + dockerfile: Dockerfile-sentiment + environment: + - REDIS_URL="redis://redis" + - REDIS_PORT=6379 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + command: ['python', 'app/models/SentimentV1/sentiment_infer_server.py'] volumes: - ./src:/opt/src networks: @@ -36,7 +56,7 @@ services: depends_on: - michaniki_client - redis - + celery_worker: build: ./src command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] @@ -54,14 +74,13 @@ services: - DB_PASSWORD=michaniki - DB_NAME=michanikidb - BROKER_URL=redis://redis:6379/0 - + redis: image: redis:4.0.5-alpine command: ["redis-server", "--appendonly", "yes"] hostname: redis networks: - michaniki - + networks: michaniki: - \ No newline at end of file diff --git a/src/Dockerfile-sentiment b/src/Dockerfile-sentiment new file mode 100644 index 0000000..5b5c09a --- /dev/null +++ b/src/Dockerfile-sentiment @@ -0,0 +1,23 @@ +FROM continuumio/miniconda3:4.5.12 + +# utils +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils + +RUN conda install gxx_linux-64 + +RUN conda install python=3.6 + +RUN apt-get install -y --force-yes default-libmysqlclient-dev mysql-client build-essential + +# Grab requirements.txt. +COPY requirementssenti.txt /tmp/requirementssenti.txt + +# Install dependencies +RUN pip install -qr /tmp/requirementssenti.txt + +# create a user for web server +RUN adduser --disabled-password --gecos "" foo + +COPY ./ /opt/src + +WORKDIR /opt/src diff --git a/src/app/apis/InceptionV3/API_helpers.py b/src/app/apis/InceptionV3/API_helpers.py index 2beef9b..44079ff 100644 --- a/src/app/apis/InceptionV3/API_helpers.py +++ b/src/app/apis/InceptionV3/API_helpers.py @@ -40,27 +40,27 @@ def save_classes_label_dict(label_dict, file_path_name): """ with open(file_path_name, 'w') as fp: json.dump(label_dict, fp) - + print "* Helper: Classes Label Json Saved" def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): """ - download the folder from S3 - + download the folder from S3 + local: /src/tmp/model_data/ - + Will not download if the local folder already exists """ print "* Helper: Loading Images from S3 {} {}".format(bucket_name,bucket_prefix) output_path = os.path.join(local_path, bucket_prefix) - + if not os.path.exists(os.path.join(output_path, 'train')): s3 = boto3.resource('s3') mybucket = s3.Bucket(bucket_name) # if blank prefix is given, return everything) objs = mybucket.objects.filter( Prefix = bucket_prefix) - + for obj in objs: path, filename = os.path.split(obj.key) save_path = os.path.join(local_path, path) @@ -69,12 +69,10 @@ def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): os.makedirs(save_path) except OSError: pass - mybucket.download_file(obj.key, os.path.join(save_path, filename)) - + try: + mybucket.download_file(obj.key, os.path.join(save_path, filename)) + except OSError: + pass + print "* Helper: Images Loaded at: {}".format(output_path) return output_path - - - - - \ No newline at end of file diff --git a/src/app/apis/InceptionV3/inceptionV3.py b/src/app/apis/InceptionV3/inceptionV3.py index 8c2caa4..32dddb1 100644 --- a/src/app/apis/InceptionV3/inceptionV3.py +++ b/src/app/apis/InceptionV3/inceptionV3.py @@ -48,12 +48,12 @@ def label(): s3_bucket_name = request.form.get('s3_bucket_name') s3_bucket_prefix = request.form.get('s3_bucket_prefix') model_name = request.form.get('model_name') - + # load image from s3 image_data_path = API_helpers.download_a_dir_from_s3(s3_bucket_name, - s3_bucket_prefix, + s3_bucket_prefix, local_path = TEMP_FOLDER) - + # for each images in the folder # supports .png and .jpg all_image_ids = [] @@ -62,28 +62,28 @@ def label(): for each_image in glob.glob(image_data_path + "/*.*"): iamge_name = each_image.split('/')[-1] this_img = image.load_img(each_image, target_size = (299, 299)) - + # image pre-processing x = np.expand_dims(image.img_to_array(this_img), axis=0) x = preprocess_input(x) x = x.copy(order="C") - + # encode x = API_helpers.base64_encode_image(x) # create a image id this_id = str(uuid.uuid4()) all_image_ids.append((this_id, iamge_name)) d = {"id": this_id, "image": x, "model_name": model_name} - + # push to the redis queue db.rpush(INCEPTIONV3_IMAGE_QUEUE, json.dumps(d)) - + all_pred = [] while all_image_ids: # pop the first one from the queue this_id, this_image_name = all_image_ids.pop(0) this_pred = {} - + while True: # check if the response has been returned output = db.get(this_id) @@ -92,27 +92,27 @@ def label(): this_pred["image name"] = this_image_name output = output.decode('utf-8') this_pred["prediction"] = json.loads(output) - + db.delete(this_id) break else: time.sleep(CLIENT_SLEEP) - + all_pred.append(this_pred) - + # remove the temp folder shutil.rmtree(image_data_path, ignore_errors=True) - + return jsonify({ "data": all_pred - }) - + }) + @blueprint.route('/retrain', methods=['POST']) def retrain(): """ pick up a pre-trained model resume training using more data - + @args: train_bucket_url: URL pointing to the folder for training data on S3 @args: model_name: the name of the model want to be retraiend, the folder must be exsit """ @@ -120,15 +120,15 @@ def retrain(): s3_bucket_prefix = request.form.get('train_bucket_prefix') nb_epoch = request.form.get('nb_epoch') batch_size = request.form.get('batch_size') - + model_name = s3_bucket_prefix.split('/')[-1] local_data_path = os.path.join('./tmp') - + # create a celer task id this_id = celery.uuid() # download the folder in the url # return the path of the image files - async_retrain.apply_async((model_name, + async_retrain.apply_async((model_name, local_data_path, s3_bucket_name, s3_bucket_prefix, @@ -140,46 +140,46 @@ def retrain(): "task_id": this_id, "status": "Retraining and Fine-Tuning are Initiated" }), 200 - + @blueprint.route('/transfer', methods=['POST']) def init_new_model(): """ init a new model based on InceptionV3 that can predict picture for new classes. - + @args: train_bucket_url: URL pointing to the folder for training data on S3 """ # need to load the base model here s3_bucket_name = request.form.get('train_bucket_name') s3_bucket_prefix = request.form.get('train_bucket_prefix') model_name = s3_bucket_prefix.split('/')[-1] - + # generate a celery task id this_id = celery.uuid() # download the folder in the url # kick off the transfer learning thing here - async_transfer.apply_async((model_name, + async_transfer.apply_async((model_name, s3_bucket_name, s3_bucket_prefix, this_id), task_id=this_id) - + return jsonify({ "task_id": this_id, "status": "Transfer Learning and Fine-Tuning are Initiated" }), 200 - + @blueprint.route('/predict', methods=['POST']) def run_inceptionV3(): """ - Run the pre-trained base Inception V3 model + Run the pre-trained base Inception V3 model and send image to queue - - Listening user submitted images and + + Listening user submitted images and stack them in a Redis queue """ data = {"success": False} - + # load model name model_name = request.form.get('model_name') @@ -189,41 +189,33 @@ def run_inceptionV3(): x = np.expand_dims(image.img_to_array(img), axis=0) x = preprocess_input(x) x = x.copy(order="C") - + # encode x = API_helpers.base64_encode_image(x) # create a image id this_id = str(uuid.uuid4()) - + d = {"id": this_id, "image": x, "model_name": model_name} - + # push to the redis queue db.rpush(INCEPTIONV3_IMAGE_QUEUE, json.dumps(d)) while True: # check if the response has been returned output = db.get(this_id) - + if output is not None: output = output.decode('utf-8') data["prediction"] = json.loads(output) - + db.delete(this_id) break else: # print "* Waiting for the Inference Server..." time.sleep(CLIENT_SLEEP) - + data['success'] = True - + return jsonify({ "data": data }), 200 - - - - - - - - \ No newline at end of file diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py new file mode 100644 index 0000000..3f298f3 --- /dev/null +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -0,0 +1,63 @@ +''' +Created on Jun 8, 2018 + +@author: runshengsong +''' +# TO DO +# move settings to config +import os +import sys +import glob +import json +import boto3 +import base64 +import numpy as np +import logging + +from keras.preprocessing import image + + +def save_classes_label_dict(label_dict, file_path_name): + """ + save the class label dictionary to json + """ + with open(file_path_name, 'w') as fp: + json.dump(label_dict, fp) + + logging.info("* Helper: Classes Label Json Saved") + +def download_a_dir_from_s3(bucket_name, bucket_prefix, local_path): + """ + download the folder from S3 + + local: /src/tmp/model_data/ + + Will not download if the local folder already exists + """ + logging.info("* Helper: Loading Text from S3 {} {}".format(bucket_name,bucket_prefix)) + path = os.path.join(bucket_name,'data') + output_path = os.path.join(local_path, bucket_name) + save_path = os.path.join(local_path, path) + logging.info('*Saving text files at:%s',save_path) + + s3 = boto3.resource('s3') + mybucket = s3.Bucket(bucket_name) + # if blank prefix is given, return everything) + key1 = 'data/train.csv' + key2 = 'data/dev.csv' + key3 = 'data/test.csv' + try: + os.makedirs(save_path) + except OSError: + pass + try: + s3.Bucket(bucket_name).download_file(key1, os.path.join(save_path,'train.csv')) + s3.Bucket(bucket_name).download_file(key2, os.path.join(save_path,'dev.csv')) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "404": + print("The object does not exist.") + else: + raise + + logging.info("* Helper: Text Loaded at: {}".format(output_path)) + return output_path diff --git a/src/app/apis/SentimentV1/__init__.py b/src/app/apis/SentimentV1/__init__.py new file mode 100644 index 0000000..1d6da5b --- /dev/null +++ b/src/app/apis/SentimentV1/__init__.py @@ -0,0 +1,6 @@ +''' +Created on Jan 23, 2019 + +@author: manu +''' +from sentimentV1 import * diff --git a/src/app/apis/SentimentV1/sentimentV1.py b/src/app/apis/SentimentV1/sentimentV1.py new file mode 100644 index 0000000..5d241c9 --- /dev/null +++ b/src/app/apis/SentimentV1/sentimentV1.py @@ -0,0 +1,102 @@ +''' +Created on Jan 22, 2019 + +Web service for Sentiment Analysis + +@author: manu +''' +import uuid +import json +import os +import io +import time +import celery +import json +import logging +# flask +from flask import jsonify +from flask import Blueprint, request + +import API_helpers_nlp + +from app import app +from app import db + +# michaniki app +from ...tasks import * + +# temp folder save image files downloaded from S3 +TEMP_FOLDER = os.path.join('./tmp') + +blueprint = Blueprint('sentimentV1', __name__) + +@blueprint.route('/predict', methods=['POST']) +def pred_sentiment(): + """ + Run the pre-trained base Sentiment analysis model + and send sentence to queue + + Listening user submitted sentences and + stack them in a Redis queue + """ + + logging.info("Inside pred_Sentence") + data = {"success": False} + + model_name = 'base' + + message = request.form.get('textv') + logging.info("Received message:%s", message) + #sentence = Sentence(message) + + # create a image id + this_id = str(uuid.uuid4()) + + d = {"id": this_id, "text": message, "model_name": model_name} + + # push to the redis queue + db.rpush(SENTIMENT_TEXT_QUEUE, json.dumps(d)) + + while True: + # check if the response has been returned + output = db.get(this_id) + if output is not None: + output = output.decode('utf-8') + data["prediction"] = json.loads(output) + + db.delete(this_id) + break + else: + #print "* Waiting for the Sentiment Inference Server..." + time.sleep(CLIENT_SLEEP) + + data['success'] = True + return jsonify({ + "data": data + }), 200 + +@blueprint.route('/trainbert', methods=['POST']) +def train_bert(): + """ + Finetune BERT uncased small language model + """ + s3_bucket_name = request.form.get('s3_bucket_name') + model_name = request.form.get('model_name') + local_data_path = os.path.join('./tmp') + batch_size = 32 + nb_epoch = 3 + + # create a celer task id + this_id = celery.uuid() + + async_berttrain.apply_async((model_name, + local_data_path, + s3_bucket_name, + s3_bucket_prefix, + nb_epoch, + batch_size, + this_id), task_id=this_id) + return jsonify({ + "task_id": this_id, + "status": "Retraining and Fine-Tuning usign BERT is Initiated" + }), 200 diff --git a/src/app/apis/apis.py b/src/app/apis/apis.py index 59ddc2b..08dcb84 100644 --- a/src/app/apis/apis.py +++ b/src/app/apis/apis.py @@ -9,6 +9,7 @@ from .mnist import blueprint as mnist_blueprint from .InceptionV3 import blueprint as incept_blueprint +from .SentimentV1 import blueprint as sentiment_blueprint from .tasks import blueprint as tasks_blueprint from app import app @@ -16,6 +17,7 @@ app.register_blueprint(mnist_blueprint, url_prefix = '/mnist') app.register_blueprint(incept_blueprint, url_prefix = '/inceptionV3') app.register_blueprint(tasks_blueprint, url_prefix='/tasks') +app.register_blueprint(sentiment_blueprint, url_prefix = '/sentimentV1') @app.route('/') def index(): @@ -24,5 +26,3 @@ def index(): @app.route('/add') def add_a(): res = add.delay(3, 4) - - diff --git a/src/app/celeryapp.py b/src/app/celeryapp.py index 3d558ff..4dfb9f7 100644 --- a/src/app/celeryapp.py +++ b/src/app/celeryapp.py @@ -4,13 +4,12 @@ from . import app BROKER_URL = app.config['BROKER_URL'] -BACKEND_URL=app.config['BACKEND_URL'] +BACKEND_URL = app.config['BACKEND_URL'] -michaniki_celery_app = celery.Celery('app', +michaniki_celery_app = celery.Celery('app', broker=BROKER_URL, backend=BACKEND_URL, include='app.tasks') if __name__ == '__main__': michaniki_celery_app.start() - \ No newline at end of file diff --git a/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py b/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py index f65c638..e46f051 100644 --- a/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py +++ b/src/app/models/InceptionV3/inceptionV3_transfer_retraining.py @@ -5,7 +5,7 @@ ''' import os import glob - +import logging import keras from keras.models import Model @@ -24,11 +24,11 @@ class InceptionRetrainer: def __init__(self, model_name): self.model_name = model_name - - def retrain(self, - this_model, - local_dir, - nb_epoch, + + def retrain(self, + this_model, + local_dir, + nb_epoch, batch_size): """ retrain the model @@ -36,35 +36,35 @@ def retrain(self, # load the training data train_dir = os.path.join(local_dir, "train") val_dir = os.path.join(local_dir, "val") - + # set up parameters nb_train_samples = self.__get_nb_files(train_dir) nb_classes = len(glob.glob(train_dir + "/*")) nb_val_samples = self.__get_nb_files(val_dir) nb_epoch = int(nb_epoch) batch_size = int(batch_size) - + # set up image data train_datagen = ImageDataGenerator( preprocessing_function = preprocess_input ) - + val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input ) - + # generator train_generator = train_datagen.flow_from_directory( train_dir, target_size=(299, 299), batch_size=batch_size) - + validation_generator = val_datagen.flow_from_directory( val_dir, target_size=(299, 299), batch_size=batch_size, ) - + # retrain the model retrain_history = this_model.fit_generator(train_generator, nb_epoch=nb_epoch, @@ -75,9 +75,10 @@ def retrain(self, verbose=1) return this_model, retrain_history - + def __get_nb_files(self, directory): """Get number of files by searching local dir recursively""" + logging.info("Inside __get_nb_files") if not os.path.exists(directory): return 0 cnt = 0 @@ -85,7 +86,7 @@ def __get_nb_files(self, directory): for dr in dirs: cnt += len(glob.glob(os.path.join(r, dr + "/*"))) return cnt - + class InceptionTransferLeaner: def __init__(self, model_name): self.model_name = model_name @@ -94,13 +95,14 @@ def __init__(self, model_name): try: print "* Transfer: Loading Topless Model..." self.topless_model = load_model(INCEPTIONV3_TOPLESS_MODEL_PATH) + except IOError: # load model from keras print "* Transfer: Loading Topless Model from Keras..." - self.topless_model = InceptionV3(include_top=False, + self.topless_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(299, 299, 3)) - + self.new_model = None # init the new model def transfer_model(self, local_dir, @@ -110,48 +112,55 @@ def transfer_model(self, local_dir, transfer the topless InceptionV3 model to classify new classes """ + + + train_dir = os.path.join(local_dir, "train") val_dir = os.path.join(local_dir, "val") - + + + # set up parameters nb_train_samples = self.__get_nb_files(train_dir) nb_classes = len(glob.glob(train_dir + "/*")) nb_val_samples = self.__get_nb_files(val_dir) nb_epoch = int(nb_epoch) batch_size = int(batch_size) - + + + # data prep train_datagen = ImageDataGenerator( preprocessing_function = preprocess_input ) - + val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input ) - + # generator train_generator = train_datagen.flow_from_directory( train_dir, target_size=(299, 299), batch_size=batch_size) - + validation_generator = val_datagen.flow_from_directory( val_dir, target_size=(299, 299), batch_size=batch_size, ) - + # get the class and label name, reverse key and value pair classes_label_dict = train_generator.class_indices classes_label_dict = {v: k for k, v in classes_label_dict.iteritems()} - + # add a new top layer base on the user data - self.new_model = self.__add_new_last_layer(self.topless_model, nb_classes) - + self.new_model = self.__add_new_last_layer(self.topless_model, nb_classes) + # set up transfer learning model - self.__setup_to_transfer_learn(model=self.new_model, + self.__setup_to_transfer_learn(model=self.new_model, base_model=self.topless_model) - + print "* Transfer: Added a New Last Layer... Starting Transfer Learning..." # train the new model for few epoch # TO DO: @@ -163,10 +172,10 @@ def transfer_model(self, local_dir, nb_val_samples=nb_val_samples//batch_size, class_weight='auto', verbose=2) - + # set up fine-tuning model self.__setup_to_finetune(self.new_model, nb_layer_to_freeze=10) - + print "* Transfer: Starting Fine-Tuning..." # train the new model again to fine-tune it history_ft = self.new_model.fit_generator(train_generator, @@ -179,7 +188,7 @@ def transfer_model(self, local_dir, # return the model return self.new_model, classes_label_dict, history_ft - + def __setup_to_finetune(self, model, nb_layer_to_freeze): """ Freeze the bottom NB_IV3_LAYERS and retrain the remaining top layers. @@ -198,7 +207,7 @@ def __setup_to_transfer_learn(self, model, base_model): for layer in base_model.layers: layer.trainable = False model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) - + def __add_new_last_layer(self, topless_model, nb_classes): """ add the last layer to the topless model @@ -209,9 +218,10 @@ def __add_new_last_layer(self, topless_model, nb_classes): predictions = Dense(nb_classes, activation='softmax')(x) #new softmax layer model = Model(input=topless_model.input, output=predictions) return model - + def __get_nb_files(self, directory): """Get number of files by searching local dir recursively""" + if not os.path.exists(directory): return 0 cnt = 0 diff --git a/src/app/models/InceptionV3/inception_inference_server.py b/src/app/models/InceptionV3/inception_inference_server.py index c118c5b..bb191bf 100644 --- a/src/app/models/InceptionV3/inception_inference_server.py +++ b/src/app/models/InceptionV3/inception_inference_server.py @@ -31,15 +31,15 @@ class inceptionV3_infernece_server: def __init__(self): # pre-load some models here on start self.loaded_models = {} - + def run_inceptionV3_infernece_server(self): ''' run the inference server for Inception V3 - - Pull image from the Redis, decode + + Pull image from the Redis, decode send to the model, predict return the response to the redis - + Images are tracked using is Image IDs ''' while True: @@ -47,15 +47,15 @@ def run_inceptionV3_infernece_server(self): imageIDs = defaultdict(list) batch_for_each_model = defaultdict(partial(np.ndarray, 0)) num_pic = 0 - + for q in queue: q = json.loads(q.decode("utf-8")) - - # decode image - this_image = INV3_helpers.base64_decode_image(q['image'], + + # decode image + this_image = INV3_helpers.base64_decode_image(q['image'], settings.IMAGE_TYPE, shape = settings.IMAGE_SHAPE) - + model_to_go = str(q['model_name']) # stack up the image to the batch # for each model @@ -66,20 +66,20 @@ def run_inceptionV3_infernece_server(self): # vstack on it batch_for_each_model[model_to_go] = np.vstack([batch_for_each_model[model_to_go], this_image]) - + # add the image id imageIDs[model_to_go].append(q['id']) num_pic += 1 - + # if there is any images in the batch if imageIDs: print "* Predicting for {} of Models".format(len(imageIDs.keys())) print "* Number of Picture: {}".format(num_pic) - + # loop over each model and predict their batch for each_model_name, each_batch in batch_for_each_model.iteritems(): this_ids = imageIDs[each_model_name] # these are the ids for the batch for this model - + # load model here # check the model if already exsit if each_model_name in self.loaded_models.keys(): @@ -91,39 +91,37 @@ def run_inceptionV3_infernece_server(self): model = load_model(os.path.join(settings.InceptionV3_MODEL_PATH, each_model_name, each_model_name+'.h5')) self.loaded_models[each_model_name] = model# save the model instance print "* {} Loaded and Saved in Mem.".format(each_model_name) - + # start predicting preds = model.predict(each_batch) - + # TO DO: # Decode prediction to get the class label results = INV3_helpers.decode_pred_to_label(preds, each_model_name, num_return = settings.NUM_LABEL_TO_RETURN) - + # loop ever each image in the batch for (each_id, each_result) in zip(this_ids, results): this_output = [] - + # generate probability of top classes for (label, prob) in each_result: r = {"label": label, "probability": float(prob)} - + this_output.append(r) - + # add this result to the queue # indexed by image id db.set(each_id, json.dumps(this_output)) print "* Prediction for {} Sent Back!".format(each_model_name) - + # delete this image batch from the queue # to save space db.ltrim(settings.IMAGE_QUEUE, len(imageIDs), -1) - + # sleep and wait time.sleep(settings.SERVER_SLEEP) - + if __name__ == "__main__": this_server = inceptionV3_infernece_server() this_server.run_inceptionV3_infernece_server() - - \ No newline at end of file diff --git a/src/app/models/InceptionV3/settings.py b/src/app/models/InceptionV3/settings.py index 9782495..7e92bca 100644 --- a/src/app/models/InceptionV3/settings.py +++ b/src/app/models/InceptionV3/settings.py @@ -7,4 +7,4 @@ IMAGE_SHAPE = (1, 299, 299, 3) FC_SIZE = 1024 SERVER_SLEEP = 0.5 -NUM_LABEL_TO_RETURN = 5 \ No newline at end of file +NUM_LABEL_TO_RETURN = 5 diff --git a/src/app/models/SentimentV1/__init__.py b/src/app/models/SentimentV1/__init__.py new file mode 100644 index 0000000..0a32dd6 --- /dev/null +++ b/src/app/models/SentimentV1/__init__.py @@ -0,0 +1 @@ +from .sentimentV1_transfer_retraining import * diff --git a/src/app/models/SentimentV1/modeling.py b/src/app/models/SentimentV1/modeling.py new file mode 100644 index 0000000..8a951f5 --- /dev/null +++ b/src/app/models/SentimentV1/modeling.py @@ -0,0 +1,988 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The main BERT model and related functions.""" + +#from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import json +import math +import re +import six +import tensorflow as tf + + +class BertConfig(object): + """Configuration for `BertModel`.""" + + def __init__(self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with tf.gfile.GFile(json_file, "r") as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +class BertModel(object): + """BERT model ("Bidirectional Encoder Representations from Transformers"). + + Example usage: + + ```python + # Already been converted into WordPiece token ids + input_ids = tf.constant([[31, 51, 99], [15, 5, 0]]) + input_mask = tf.constant([[1, 1, 1], [1, 1, 0]]) + token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]]) + + config = modeling.BertConfig(vocab_size=32000, hidden_size=512, + num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) + + model = modeling.BertModel(config=config, is_training=True, + input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids) + + label_embeddings = tf.get_variable(...) + pooled_output = model.get_pooled_output() + logits = tf.matmul(pooled_output, label_embeddings) + ... + ``` + """ + + def __init__(self, + config, + is_training, + input_ids, + input_mask=None, + token_type_ids=None, + use_one_hot_embeddings=True, + scope=None): + """Constructor for BertModel. + + Args: + config: `BertConfig` instance. + is_training: bool. true for training model, false for eval model. Controls + whether dropout will be applied. + input_ids: int32 Tensor of shape [batch_size, seq_length]. + input_mask: (optional) int32 Tensor of shape [batch_size, seq_length]. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + use_one_hot_embeddings: (optional) bool. Whether to use one-hot word + embeddings or tf.embedding_lookup() for the word embeddings. On the TPU, + it is much faster if this is True, on the CPU or GPU, it is faster if + this is False. + scope: (optional) variable scope. Defaults to "bert". + + Raises: + ValueError: The config is invalid or one of the input tensor shapes + is invalid. + """ + config = copy.deepcopy(config) + if not is_training: + config.hidden_dropout_prob = 0.0 + config.attention_probs_dropout_prob = 0.0 + + input_shape = get_shape_list(input_ids, expected_rank=2) + batch_size = input_shape[0] + seq_length = input_shape[1] + + if input_mask is None: + input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32) + + if token_type_ids is None: + token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32) + + with tf.variable_scope(scope, default_name="bert"): + with tf.variable_scope("embeddings"): + # Perform embedding lookup on the word ids. + (self.embedding_output, self.embedding_table) = embedding_lookup( + input_ids=input_ids, + vocab_size=config.vocab_size, + embedding_size=config.hidden_size, + initializer_range=config.initializer_range, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=use_one_hot_embeddings) + + # Add positional embeddings and token type embeddings, then layer + # normalize and perform dropout. + self.embedding_output = embedding_postprocessor( + input_tensor=self.embedding_output, + use_token_type=True, + token_type_ids=token_type_ids, + token_type_vocab_size=config.type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=config.initializer_range, + max_position_embeddings=config.max_position_embeddings, + dropout_prob=config.hidden_dropout_prob) + + with tf.variable_scope("encoder"): + # This converts a 2D mask of shape [batch_size, seq_length] to a 3D + # mask of shape [batch_size, seq_length, seq_length] which is used + # for the attention scores. + attention_mask = create_attention_mask_from_input_mask( + input_ids, input_mask) + + # Run the stacked transformer. + # `sequence_output` shape = [batch_size, seq_length, hidden_size]. + self.all_encoder_layers = transformer_model( + input_tensor=self.embedding_output, + attention_mask=attention_mask, + hidden_size=config.hidden_size, + num_hidden_layers=config.num_hidden_layers, + num_attention_heads=config.num_attention_heads, + intermediate_size=config.intermediate_size, + intermediate_act_fn=get_activation(config.hidden_act), + hidden_dropout_prob=config.hidden_dropout_prob, + attention_probs_dropout_prob=config.attention_probs_dropout_prob, + initializer_range=config.initializer_range, + do_return_all_layers=True) + + self.sequence_output = self.all_encoder_layers[-1] + # The "pooler" converts the encoded sequence tensor of shape + # [batch_size, seq_length, hidden_size] to a tensor of shape + # [batch_size, hidden_size]. This is necessary for segment-level + # (or segment-pair-level) classification tasks where we need a fixed + # dimensional representation of the segment. + with tf.variable_scope("pooler"): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. We assume that this has been pre-trained + first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1) + self.pooled_output = tf.layers.dense( + first_token_tensor, + config.hidden_size, + activation=tf.tanh, + kernel_initializer=create_initializer(config.initializer_range)) + + def get_pooled_output(self): + return self.pooled_output + + def get_sequence_output(self): + """Gets final hidden layer of encoder. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the final hidden of the transformer encoder. + """ + return self.sequence_output + + def get_all_encoder_layers(self): + return self.all_encoder_layers + + def get_embedding_output(self): + """Gets output of the embedding lookup (i.e., input to the transformer). + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the output of the embedding layer, after summing the word + embeddings with the positional embeddings and the token type embeddings, + then performing layer normalization. This is the input to the transformer. + """ + return self.embedding_output + + def get_embedding_table(self): + return self.embedding_table + + +def gelu(input_tensor): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + + Args: + input_tensor: float Tensor to perform activation. + + Returns: + `input_tensor` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.erf(input_tensor / tf.sqrt(2.0))) + return input_tensor * cdf + + +def get_activation(activation_string): + """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`. + + Args: + activation_string: String name of the activation function. + + Returns: + A Python function corresponding to the activation function. If + `activation_string` is None, empty, or "linear", this will return None. + If `activation_string` is not a string, it will return `activation_string`. + + Raises: + ValueError: The `activation_string` does not correspond to a known + activation. + """ + + # We assume that anything that"s not a string is already an activation + # function, so we just return it. + if not isinstance(activation_string, six.string_types): + return activation_string + + if not activation_string: + return None + + act = activation_string.lower() + if act == "linear": + return None + elif act == "relu": + return tf.nn.relu + elif act == "gelu": + return gelu + elif act == "tanh": + return tf.tanh + else: + raise ValueError("Unsupported activation: %s" % act) + + +def get_assignment_map_from_checkpoint(tvars, init_checkpoint): + """Compute the union of the current variables and checkpoint variables.""" + assignment_map = {} + initialized_variable_names = {} + + name_to_variable = collections.OrderedDict() + for var in tvars: + name = var.name + m = re.match("^(.*):\\d+$", name) + if m is not None: + name = m.group(1) + name_to_variable[name] = var + + init_vars = tf.train.list_variables(init_checkpoint) + + assignment_map = collections.OrderedDict() + for x in init_vars: + (name, var) = (x[0], x[1]) + if name not in name_to_variable: + continue + assignment_map[name] = name + initialized_variable_names[name] = 1 + initialized_variable_names[name + ":0"] = 1 + + return (assignment_map, initialized_variable_names) + + +def dropout(input_tensor, dropout_prob): + """Perform dropout. + + Args: + input_tensor: float Tensor. + dropout_prob: Python float. The probability of dropping out a value (NOT of + *keeping* a dimension as in `tf.nn.dropout`). + + Returns: + A version of `input_tensor` with dropout applied. + """ + if dropout_prob is None or dropout_prob == 0.0: + return input_tensor + + output = tf.nn.dropout(input_tensor, 1.0 - dropout_prob) + return output + + +def layer_norm(input_tensor, name=None): + """Run layer normalization on the last dimension of the tensor.""" + return tf.contrib.layers.layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + + +def layer_norm_and_dropout(input_tensor, dropout_prob, name=None): + """Runs layer normalization followed by dropout.""" + output_tensor = layer_norm(input_tensor, name) + output_tensor = dropout(output_tensor, dropout_prob) + return output_tensor + + +def create_initializer(initializer_range=0.02): + """Creates a `truncated_normal_initializer` with the given range.""" + return tf.truncated_normal_initializer(stddev=initializer_range) + + +def embedding_lookup(input_ids, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=False): + """Looks up words embeddings for id tensor. + + Args: + input_ids: int32 Tensor of shape [batch_size, seq_length] containing word + ids. + vocab_size: int. Size of the embedding vocabulary. + embedding_size: int. Width of the word embeddings. + initializer_range: float. Embedding initialization range. + word_embedding_name: string. Name of the embedding table. + use_one_hot_embeddings: bool. If True, use one-hot method for word + embeddings. If False, use `tf.nn.embedding_lookup()`. One hot is better + for TPUs. + + Returns: + float Tensor of shape [batch_size, seq_length, embedding_size]. + """ + # This function assumes that the input is of shape [batch_size, seq_length, + # num_inputs]. + # + # If the input is a 2D tensor of shape [batch_size, seq_length], we + # reshape to [batch_size, seq_length, 1]. + if input_ids.shape.ndims == 2: + input_ids = tf.expand_dims(input_ids, axis=[-1]) + + embedding_table = tf.get_variable( + name=word_embedding_name, + shape=[vocab_size, embedding_size], + initializer=create_initializer(initializer_range)) + + if use_one_hot_embeddings: + flat_input_ids = tf.reshape(input_ids, [-1]) + one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) + output = tf.matmul(one_hot_input_ids, embedding_table) + else: + output = tf.nn.embedding_lookup(embedding_table, input_ids) + + input_shape = get_shape_list(input_ids) + + output = tf.reshape(output, + input_shape[0:-1] + [input_shape[-1] * embedding_size]) + return (output, embedding_table) + + +def embedding_postprocessor(input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1): + """Performs various post-processing on a word embedding tensor. + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, + embedding_size]. + use_token_type: bool. Whether to add embeddings for `token_type_ids`. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + Must be specified if `use_token_type` is True. + token_type_vocab_size: int. The vocabulary size of `token_type_ids`. + token_type_embedding_name: string. The name of the embedding table variable + for token type ids. + use_position_embeddings: bool. Whether to add position embeddings for the + position of each token in the sequence. + position_embedding_name: string. The name of the embedding table variable + for positional embeddings. + initializer_range: float. Range of the weight initialization. + max_position_embeddings: int. Maximum sequence length that might ever be + used with this model. This can be longer than the sequence length of + input_tensor, but cannot be shorter. + dropout_prob: float. Dropout probability applied to the final output tensor. + + Returns: + float tensor with same shape as `input_tensor`. + + Raises: + ValueError: One of the tensor shapes or input values is invalid. + """ + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + width = input_shape[2] + + output = input_tensor + + if use_token_type: + if token_type_ids is None: + raise ValueError("`token_type_ids` must be specified if" + "`use_token_type` is True.") + token_type_table = tf.get_variable( + name=token_type_embedding_name, + shape=[token_type_vocab_size, width], + initializer=create_initializer(initializer_range)) + # This vocab will be small so we always do one-hot here, since it is always + # faster for a small vocabulary. + flat_token_type_ids = tf.reshape(token_type_ids, [-1]) + one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) + token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) + token_type_embeddings = tf.reshape(token_type_embeddings, + [batch_size, seq_length, width]) + output += token_type_embeddings + + if use_position_embeddings: + assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) + with tf.control_dependencies([assert_op]): + full_position_embeddings = tf.get_variable( + name=position_embedding_name, + shape=[max_position_embeddings, width], + initializer=create_initializer(initializer_range)) + # Since the position embedding table is a learned variable, we create it + # using a (long) sequence length `max_position_embeddings`. The actual + # sequence length might be shorter than this, for faster training of + # tasks that do not have long sequences. + # + # So `full_position_embeddings` is effectively an embedding table + # for position [0, 1, 2, ..., max_position_embeddings-1], and the current + # sequence has positions [0, 1, 2, ... seq_length-1], so we can just + # perform a slice. + position_embeddings = tf.slice(full_position_embeddings, [0, 0], + [seq_length, -1]) + num_dims = len(output.shape.as_list()) + + # Only the last two dimensions are relevant (`seq_length` and `width`), so + # we broadcast among the first dimensions, which is typically just + # the batch size. + position_broadcast_shape = [] + for _ in range(num_dims - 2): + position_broadcast_shape.append(1) + position_broadcast_shape.extend([seq_length, width]) + position_embeddings = tf.reshape(position_embeddings, + position_broadcast_shape) + output += position_embeddings + + output = layer_norm_and_dropout(output, dropout_prob) + return output + + +def create_attention_mask_from_input_mask(from_tensor, to_mask): + """Create 3D attention mask from a 2D tensor mask. + + Args: + from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...]. + to_mask: int32 Tensor of shape [batch_size, to_seq_length]. + + Returns: + float Tensor of shape [batch_size, from_seq_length, to_seq_length]. + """ + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + batch_size = from_shape[0] + from_seq_length = from_shape[1] + + to_shape = get_shape_list(to_mask, expected_rank=2) + to_seq_length = to_shape[1] + + to_mask = tf.cast( + tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) + + # We don't assume that `from_tensor` is a mask (although it could be). We + # don't actually care if we attend *from* padding tokens (only *to* padding) + # tokens so we create a tensor of all ones. + # + # `broadcast_ones` = [batch_size, from_seq_length, 1] + broadcast_ones = tf.ones( + shape=[batch_size, from_seq_length, 1], dtype=tf.float32) + + # Here we broadcast along two dimensions to create the mask. + mask = broadcast_ones * to_mask + + return mask + + +def attention_layer(from_tensor, + to_tensor, + attention_mask=None, + num_attention_heads=1, + size_per_head=512, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None): + """Performs multi-headed attention from `from_tensor` to `to_tensor`. + + This is an implementation of multi-headed attention based on "Attention + is all you Need". If `from_tensor` and `to_tensor` are the same, then + this is self-attention. Each timestep in `from_tensor` attends to the + corresponding sequence in `to_tensor`, and returns a fixed-with vector. + + This function first projects `from_tensor` into a "query" tensor and + `to_tensor` into "key" and "value" tensors. These are (effectively) a list + of tensors of length `num_attention_heads`, where each tensor is of shape + [batch_size, seq_length, size_per_head]. + + Then, the query and key tensors are dot-producted and scaled. These are + softmaxed to obtain attention probabilities. The value tensors are then + interpolated by these probabilities, then concatenated back to a single + tensor and returned. + + In practice, the multi-headed attention are done with transposes and + reshapes rather than actual separate tensors. + + Args: + from_tensor: float Tensor of shape [batch_size, from_seq_length, + from_width]. + to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width]. + attention_mask: (optional) int32 Tensor of shape [batch_size, + from_seq_length, to_seq_length]. The values should be 1 or 0. The + attention scores will effectively be set to -infinity for any positions in + the mask that are 0, and will be unchanged for positions that are 1. + num_attention_heads: int. Number of attention heads. + size_per_head: int. Size of each attention head. + query_act: (optional) Activation function for the query transform. + key_act: (optional) Activation function for the key transform. + value_act: (optional) Activation function for the value transform. + attention_probs_dropout_prob: (optional) float. Dropout probability of the + attention probabilities. + initializer_range: float. Range of the weight initializer. + do_return_2d_tensor: bool. If True, the output will be of shape [batch_size + * from_seq_length, num_attention_heads * size_per_head]. If False, the + output will be of shape [batch_size, from_seq_length, num_attention_heads + * size_per_head]. + batch_size: (Optional) int. If the input is 2D, this might be the batch size + of the 3D version of the `from_tensor` and `to_tensor`. + from_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `from_tensor`. + to_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `to_tensor`. + + Returns: + float Tensor of shape [batch_size, from_seq_length, + num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is + true, this will be of shape [batch_size * from_seq_length, + num_attention_heads * size_per_head]). + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + + def transpose_for_scores(input_tensor, batch_size, num_attention_heads, + seq_length, width): + output_tensor = tf.reshape( + input_tensor, [batch_size, seq_length, num_attention_heads, width]) + + output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) + return output_tensor + + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) + + if len(from_shape) != len(to_shape): + raise ValueError( + "The rank of `from_tensor` must match the rank of `to_tensor`.") + + if len(from_shape) == 3: + batch_size = from_shape[0] + from_seq_length = from_shape[1] + to_seq_length = to_shape[1] + elif len(from_shape) == 2: + if (batch_size is None or from_seq_length is None or to_seq_length is None): + raise ValueError( + "When passing in rank 2 tensors to attention_layer, the values " + "for `batch_size`, `from_seq_length`, and `to_seq_length` " + "must all be specified.") + + # Scalar dimensions referenced here: + # B = batch size (number of sequences) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + + from_tensor_2d = reshape_to_matrix(from_tensor) + to_tensor_2d = reshape_to_matrix(to_tensor) + + # `query_layer` = [B*F, N*H] + query_layer = tf.layers.dense( + from_tensor_2d, + num_attention_heads * size_per_head, + activation=query_act, + name="query", + kernel_initializer=create_initializer(initializer_range)) + + # `key_layer` = [B*T, N*H] + key_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=key_act, + name="key", + kernel_initializer=create_initializer(initializer_range)) + + # `value_layer` = [B*T, N*H] + value_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=value_act, + name="value", + kernel_initializer=create_initializer(initializer_range)) + + # `query_layer` = [B, N, F, H] + query_layer = transpose_for_scores(query_layer, batch_size, + num_attention_heads, from_seq_length, + size_per_head) + + # `key_layer` = [B, N, T, H] + key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, + to_seq_length, size_per_head) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + # `attention_scores` = [B, N, F, T] + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(size_per_head))) + + if attention_mask is not None: + # `attention_mask` = [B, 1, F, T] + attention_mask = tf.expand_dims(attention_mask, axis=[1]) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + adder = (1.0 - tf.cast(attention_mask, tf.float32)) * -10000.0 + + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + attention_scores += adder + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, N, F, T] + attention_probs = tf.nn.softmax(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = dropout(attention_probs, attention_probs_dropout_prob) + + # `value_layer` = [B, T, N, H] + value_layer = tf.reshape( + value_layer, + [batch_size, to_seq_length, num_attention_heads, size_per_head]) + + # `value_layer` = [B, N, T, H] + value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) + + # `context_layer` = [B, N, F, H] + context_layer = tf.matmul(attention_probs, value_layer) + + # `context_layer` = [B, F, N, H] + context_layer = tf.transpose(context_layer, [0, 2, 1, 3]) + + if do_return_2d_tensor: + # `context_layer` = [B*F, N*H] + context_layer = tf.reshape( + context_layer, + [batch_size * from_seq_length, num_attention_heads * size_per_head]) + else: + # `context_layer` = [B, F, N*H] + context_layer = tf.reshape( + context_layer, + [batch_size, from_seq_length, num_attention_heads * size_per_head]) + + return context_layer + + +def transformer_model(input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False): + """Multi-headed, multi-layer Transformer from "Attention is All You Need". + + This is almost an exact implementation of the original Transformer encoder. + + See the original paper: + https://arxiv.org/abs/1706.03762 + + Also see: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size]. + attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length, + seq_length], with 1 for positions that can be attended to and 0 in + positions that should not be. + hidden_size: int. Hidden size of the Transformer. + num_hidden_layers: int. Number of layers (blocks) in the Transformer. + num_attention_heads: int. Number of attention heads in the Transformer. + intermediate_size: int. The size of the "intermediate" (a.k.a., feed + forward) layer. + intermediate_act_fn: function. The non-linear activation function to apply + to the output of the intermediate/feed-forward layer. + hidden_dropout_prob: float. Dropout probability for the hidden layers. + attention_probs_dropout_prob: float. Dropout probability of the attention + probabilities. + initializer_range: float. Range of the initializer (stddev of truncated + normal). + do_return_all_layers: Whether to also return all layers or just the final + layer. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size], the final + hidden layer of the Transformer. + + Raises: + ValueError: A Tensor shape or parameter is invalid. + """ + if hidden_size % num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, num_attention_heads)) + + attention_head_size = int(hidden_size / num_attention_heads) + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + input_width = input_shape[2] + + # The Transformer performs sum residuals on all layers so the input needs + # to be the same as the hidden size. + if input_width != hidden_size: + raise ValueError("The width of the input tensor (%d) != hidden size (%d)" % + (input_width, hidden_size)) + + # We keep the representation as a 2D tensor to avoid re-shaping it back and + # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on + # the GPU/CPU but may not be free on the TPU, so we want to minimize them to + # help the optimizer. + prev_output = reshape_to_matrix(input_tensor) + + all_layer_outputs = [] + for layer_idx in range(num_hidden_layers): + with tf.variable_scope("layer_%d" % layer_idx): + layer_input = prev_output + + with tf.variable_scope("attention"): + attention_heads = [] + with tf.variable_scope("self"): + attention_head = attention_layer( + from_tensor=layer_input, + to_tensor=layer_input, + attention_mask=attention_mask, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=seq_length, + to_seq_length=seq_length) + attention_heads.append(attention_head) + + attention_output = None + if len(attention_heads) == 1: + attention_output = attention_heads[0] + else: + # In the case where we have other sequences, we just concatenate + # them to the self-attention head before the projection. + attention_output = tf.concat(attention_heads, axis=-1) + + # Run a linear projection of `hidden_size` then add a residual + # with `layer_input`. + with tf.variable_scope("output"): + attention_output = tf.layers.dense( + attention_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + attention_output = dropout(attention_output, hidden_dropout_prob) + attention_output = layer_norm(attention_output + layer_input) + + # The activation is only applied to the "intermediate" hidden layer. + with tf.variable_scope("intermediate"): + intermediate_output = tf.layers.dense( + attention_output, + intermediate_size, + activation=intermediate_act_fn, + kernel_initializer=create_initializer(initializer_range)) + + # Down-project back to `hidden_size` then add the residual. + with tf.variable_scope("output"): + layer_output = tf.layers.dense( + intermediate_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + layer_output = dropout(layer_output, hidden_dropout_prob) + layer_output = layer_norm(layer_output + attention_output) + prev_output = layer_output + all_layer_outputs.append(layer_output) + + if do_return_all_layers: + final_outputs = [] + for layer_output in all_layer_outputs: + final_output = reshape_from_matrix(layer_output, input_shape) + final_outputs.append(final_output) + return final_outputs + else: + final_output = reshape_from_matrix(prev_output, input_shape) + return final_output + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if name is None: + name = tensor.name + + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def reshape_to_matrix(input_tensor): + """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" + ndims = input_tensor.shape.ndims + if ndims < 2: + raise ValueError("Input tensor must have at least rank 2. Shape = %s" % + (input_tensor.shape)) + if ndims == 2: + return input_tensor + + width = input_tensor.shape[-1] + output_tensor = tf.reshape(input_tensor, [-1, width]) + return output_tensor + + +def reshape_from_matrix(output_tensor, orig_shape_list): + """Reshapes a rank 2 tensor back to its original rank >= 2 tensor.""" + if len(orig_shape_list) == 2: + return output_tensor + + output_shape = get_shape_list(output_tensor) + + orig_dims = orig_shape_list[0:-1] + width = output_shape[-1] + + return tf.reshape(output_tensor, orig_dims + [width]) + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + if name is None: + name = tensor.name + + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + scope_name = tf.get_variable_scope().name + raise ValueError( + "For the tensor `%s` in scope `%s`, the actual rank " + "`%d` (shape = %s) is not equal to the expected rank `%s`" % + (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) diff --git a/src/app/models/SentimentV1/optimization.py b/src/app/models/SentimentV1/optimization.py new file mode 100644 index 0000000..f8ccbf4 --- /dev/null +++ b/src/app/models/SentimentV1/optimization.py @@ -0,0 +1,174 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions and classes related to optimization (weight updates).""" + +#from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import tensorflow as tf + + +def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): + """Creates an optimizer training op.""" + global_step = tf.train.get_or_create_global_step() + + learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) + + # Implements linear decay of the learning rate. + learning_rate = tf.train.polynomial_decay( + learning_rate, + global_step, + num_train_steps, + end_learning_rate=0.0, + power=1.0, + cycle=False) + + # Implements linear warmup. I.e., if global_step < num_warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + if num_warmup_steps: + global_steps_int = tf.cast(global_step, tf.int32) + warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) + + global_steps_float = tf.cast(global_steps_int, tf.float32) + warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) + + warmup_percent_done = global_steps_float / warmup_steps_float + warmup_learning_rate = init_lr * warmup_percent_done + + is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) + learning_rate = ( + (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) + + # It is recommended that you use this optimizer for fine tuning, since this + # is how the model was trained (note that the Adam m/v variables are NOT + # loaded from init_checkpoint.) + optimizer = AdamWeightDecayOptimizer( + learning_rate=learning_rate, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) + + if use_tpu: + optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) + + tvars = tf.trainable_variables() + grads = tf.gradients(loss, tvars) + + # This is how the model was pre-trained. + (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) + + train_op = optimizer.apply_gradients( + zip(grads, tvars), global_step=global_step) + + # Normally the global step update is done inside of `apply_gradients`. + # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use + # a different optimizer, you should probably take this line out. + new_global_step = global_step + 1 + train_op = tf.group(train_op, [global_step.assign(new_global_step)]) + return train_op + + +class AdamWeightDecayOptimizer(tf.train.Optimizer): + """A basic Adam optimizer that includes "correct" L2 weight decay.""" + + def __init__(self, + learning_rate, + weight_decay_rate=0.0, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=None, + name="AdamWeightDecayOptimizer"): + """Constructs a AdamWeightDecayOptimizer.""" + super(AdamWeightDecayOptimizer, self).__init__(False, name) + + self.learning_rate = learning_rate + self.weight_decay_rate = weight_decay_rate + self.beta_1 = beta_1 + self.beta_2 = beta_2 + self.epsilon = epsilon + self.exclude_from_weight_decay = exclude_from_weight_decay + + def apply_gradients(self, grads_and_vars, global_step=None, name=None): + """See base class.""" + assignments = [] + for (grad, param) in grads_and_vars: + if grad is None or param is None: + continue + + param_name = self._get_variable_name(param.name) + + m = tf.get_variable( + name=param_name + "/adam_m", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + v = tf.get_variable( + name=param_name + "/adam_v", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + + # Standard Adam update. + next_m = ( + tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) + next_v = ( + tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, + tf.square(grad))) + + update = next_m / (tf.sqrt(next_v) + self.epsilon) + + # Just adding the square of the weights to the loss function is *not* + # the correct way of using L2 regularization/weight decay with Adam, + # since that will interact with the m and v parameters in strange ways. + # + # Instead we want ot decay the weights in a manner that doesn't interact + # with the m/v parameters. This is equivalent to adding the square + # of the weights to the loss with plain (non-momentum) SGD. + if self._do_use_weight_decay(param_name): + update += self.weight_decay_rate * param + + update_with_lr = self.learning_rate * update + + next_param = param - update_with_lr + + assignments.extend( + [param.assign(next_param), + m.assign(next_m), + v.assign(next_v)]) + return tf.group(*assignments, name=name) + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if not self.weight_decay_rate: + return False + if self.exclude_from_weight_decay: + for r in self.exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + def _get_variable_name(self, param_name): + """Get the variable name from the tensor name.""" + m = re.match("^(.*):\\d+$", param_name) + if m is not None: + param_name = m.group(1) + return param_name diff --git a/src/app/models/SentimentV1/run_classifier.py b/src/app/models/SentimentV1/run_classifier.py new file mode 100644 index 0000000..8da9819 --- /dev/null +++ b/src/app/models/SentimentV1/run_classifier.py @@ -0,0 +1,981 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BERT finetuning runner.""" + +#from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import csv +import os +import modeling +import optimization +import tokenization +import tensorflow as tf + +flags = tf.flags + +FLAGS = flags.FLAGS + +## Required parameters +flags.DEFINE_string( + "data_dir", None, + "The input data dir. Should contain the .tsv files (or other data files) " + "for the task.") + +flags.DEFINE_string( + "bert_config_file", None, + "The config json file corresponding to the pre-trained BERT model. " + "This specifies the model architecture.") + +flags.DEFINE_string("task_name", None, "The name of the task to train.") + +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_string( + "output_dir", None, + "The output directory where the model checkpoints will be written.") + +## Other parameters + +flags.DEFINE_string( + "init_checkpoint", None, + "Initial checkpoint (usually from a pre-trained BERT model).") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded.") + +flags.DEFINE_bool("do_train", False, "Whether to run training.") + +flags.DEFINE_bool("do_eval", False, "Whether to run eval on the dev set.") + +flags.DEFINE_bool( + "do_predict", False, + "Whether to run the model in inference mode on the test set.") + +flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") + +flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.") + +flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.") + +flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") + +flags.DEFINE_float("num_train_epochs", 3.0, + "Total number of training epochs to perform.") + +flags.DEFINE_float( + "warmup_proportion", 0.1, + "Proportion of training to perform linear learning rate warmup for. " + "E.g., 0.1 = 10% of training.") + +flags.DEFINE_integer("save_checkpoints_steps", 1000, + "How often to save the model checkpoint.") + +flags.DEFINE_integer("iterations_per_loop", 1000, + "How many steps to make in each estimator call.") + +flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") + +tf.flags.DEFINE_string( + "tpu_name", None, + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " + "url.") + +tf.flags.DEFINE_string( + "tpu_zone", None, + "[Optional] GCE zone where the Cloud TPU is located in. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") + +tf.flags.DEFINE_string( + "gcp_project", None, + "[Optional] Project name for the Cloud TPU-enabled project. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") + +tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") + +flags.DEFINE_integer( + "num_tpu_cores", 8, + "Only used if `use_tpu` is True. Total number of TPU cores to use.") + + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class PaddingInputExample(object): + """Fake example so the num input examples is a multiple of the batch size. + + When running eval/predict on the TPU, we need to pad the number of examples + to be a multiple of the batch size, because the TPU requires a fixed batch + size. The alternative is to drop the last batch, which is bad because it means + the entire output data won't be generated. + + We use this class instead of `None` because treating `None` as padding + battches could cause silent errors. + """ + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + input_ids, + input_mask, + segment_ids, + label_id, + is_real_example=True): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + self.is_real_example = is_real_example + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with tf.gfile.Open(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + +class XnliProcessor(DataProcessor): + """Processor for the XNLI data set.""" + + def __init__(self): + self.language = "zh" + + def get_train_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv( + os.path.join(data_dir, "multinli", + "multinli.train.%s.tsv" % self.language)) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "train-%d" % (i) + text_a = tokenization.convert_to_unicode(line[0]) + text_b = tokenization.convert_to_unicode(line[1]) + label = tokenization.convert_to_unicode(line[2]) + if label == tokenization.convert_to_unicode("contradictory"): + label = tokenization.convert_to_unicode("contradiction") + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "dev-%d" % (i) + language = tokenization.convert_to_unicode(line[0]) + if language != tokenization.convert_to_unicode(self.language): + continue + text_a = tokenization.convert_to_unicode(line[6]) + text_b = tokenization.convert_to_unicode(line[7]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), + "dev_matched") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, tokenization.convert_to_unicode(line[0])) + text_a = tokenization.convert_to_unicode(line[8]) + text_b = tokenization.convert_to_unicode(line[9]) + if set_type == "test": + label = "contradiction" + else: + label = tokenization.convert_to_unicode(line[-1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = tokenization.convert_to_unicode(line[3]) + text_b = tokenization.convert_to_unicode(line[4]) + if set_type == "test": + label = "0" + else: + label = tokenization.convert_to_unicode(line[0]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + # Only the test set has a header + if set_type == "test" and i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + label = "0" + else: + text_a = tokenization.convert_to_unicode(line[3]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +def convert_single_example(ex_index, example, label_list, max_seq_length, + tokenizer): + """Converts a single `InputExample` into a single `InputFeatures`.""" + + if isinstance(example, PaddingInputExample): + return InputFeatures( + input_ids=[0] * max_seq_length, + input_mask=[0] * max_seq_length, + segment_ids=[0] * max_seq_length, + label_id=0, + is_real_example=False) + + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = label_map[example.label] + if ex_index < 5: + tf.logging.info("*** Example ***") + tf.logging.info("guid: %s" % (example.guid)) + tf.logging.info("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in tokens])) + tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + tf.logging.info("label: %s (id = %d)" % (example.label, label_id)) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id, + is_real_example=True) + return feature + + +def file_based_convert_examples_to_features( + examples, label_list, max_seq_length, tokenizer, output_file): + """Convert a set of `InputExample`s to a TFRecord file.""" + + writer = tf.python_io.TFRecordWriter(output_file) + + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) + + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + features["label_ids"] = create_int_feature([feature.label_id]) + features["is_real_example"] = create_int_feature( + [int(feature.is_real_example)]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def file_based_input_fn_builder(input_file, seq_length, is_training, + drop_remainder): + """Creates an `input_fn` closure to be passed to TPUEstimator.""" + + name_to_features = { + "input_ids": tf.FixedLenFeature([seq_length], tf.int64), + "input_mask": tf.FixedLenFeature([seq_length], tf.int64), + "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), + "label_ids": tf.FixedLenFeature([], tf.int64), + "is_real_example": tf.FixedLenFeature([], tf.int64), + } + + def _decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.to_int32(t) + example[name] = t + + return example + + def input_fn(params): + """The actual input function.""" + batch_size = params["batch_size"] + + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + d = tf.data.TFRecordDataset(input_file) + if is_training: + d = d.repeat() + d = d.shuffle(buffer_size=100) + + d = d.apply( + tf.contrib.data.map_and_batch( + lambda record: _decode_record(record, name_to_features), + batch_size=batch_size, + drop_remainder=drop_remainder)) + + return d + + return input_fn + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, + labels, num_labels, use_one_hot_embeddings): + """Creates a classification model.""" + model = modeling.BertModel( + config=bert_config, + is_training=is_training, + input_ids=input_ids, + input_mask=input_mask, + token_type_ids=segment_ids, + use_one_hot_embeddings=use_one_hot_embeddings) + + # In the demo, we are doing a simple classification task on the entire + # segment. + # + # If you want to use the token-level output, use model.get_sequence_output() + # instead. + output_layer = model.get_pooled_output() + + hidden_size = output_layer.shape[-1].value + + output_weights = tf.get_variable( + "output_weights", [num_labels, hidden_size], + initializer=tf.truncated_normal_initializer(stddev=0.02)) + + output_bias = tf.get_variable( + "output_bias", [num_labels], initializer=tf.zeros_initializer()) + + with tf.variable_scope("loss"): + if is_training: + # I.e., 0.1 dropout + output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) + + logits = tf.matmul(output_layer, output_weights, transpose_b=True) + logits = tf.nn.bias_add(logits, output_bias) + probabilities = tf.nn.softmax(logits, axis=-1) + log_probs = tf.nn.log_softmax(logits, axis=-1) + + one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) + + per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) + loss = tf.reduce_mean(per_example_loss) + + return (loss, per_example_loss, logits, probabilities) + + +def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate, + num_train_steps, num_warmup_steps, use_tpu, + use_one_hot_embeddings): + """Returns `model_fn` closure for TPUEstimator.""" + + def model_fn(features, labels, mode, params): # pylint: disable=unused-argument + """The `model_fn` for TPUEstimator.""" + + tf.logging.info("*** Features ***") + for name in sorted(features.keys()): + tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) + + input_ids = features["input_ids"] + input_mask = features["input_mask"] + segment_ids = features["segment_ids"] + label_ids = features["label_ids"] + is_real_example = None + if "is_real_example" in features: + is_real_example = tf.cast(features["is_real_example"], dtype=tf.float32) + else: + is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32) + + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + (total_loss, per_example_loss, logits, probabilities) = create_model( + bert_config, is_training, input_ids, input_mask, segment_ids, label_ids, + num_labels, use_one_hot_embeddings) + + tvars = tf.trainable_variables() + initialized_variable_names = {} + scaffold_fn = None + if init_checkpoint: + (assignment_map, initialized_variable_names + ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) + if use_tpu: + + def tpu_scaffold(): + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + return tf.train.Scaffold() + + scaffold_fn = tpu_scaffold + else: + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + + tf.logging.info("**** Trainable Variables ****") + for var in tvars: + init_string = "" + if var.name in initialized_variable_names: + init_string = ", *INIT_FROM_CKPT*" + tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, + init_string) + + output_spec = None + if mode == tf.estimator.ModeKeys.TRAIN: + + train_op = optimization.create_optimizer( + total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) + + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, + loss=total_loss, + train_op=train_op, + scaffold_fn=scaffold_fn) + elif mode == tf.estimator.ModeKeys.EVAL: + + def metric_fn(per_example_loss, label_ids, logits, is_real_example): + predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) + accuracy = tf.metrics.accuracy( + labels=label_ids, predictions=predictions, weights=is_real_example) + loss = tf.metrics.mean(values=per_example_loss, weights=is_real_example) + return { + "eval_accuracy": accuracy, + "eval_loss": loss, + } + + eval_metrics = (metric_fn, + [per_example_loss, label_ids, logits, is_real_example]) + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, + loss=total_loss, + eval_metrics=eval_metrics, + scaffold_fn=scaffold_fn) + else: + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, + predictions={"probabilities": probabilities}, + scaffold_fn=scaffold_fn) + return output_spec + + return model_fn + + +# This function is not used by this file but is still used by the Colab and +# people who depend on it. +def input_fn_builder(features, seq_length, is_training, drop_remainder): + """Creates an `input_fn` closure to be passed to TPUEstimator.""" + + all_input_ids = [] + all_input_mask = [] + all_segment_ids = [] + all_label_ids = [] + + for feature in features: + all_input_ids.append(feature.input_ids) + all_input_mask.append(feature.input_mask) + all_segment_ids.append(feature.segment_ids) + all_label_ids.append(feature.label_id) + + def input_fn(params): + """The actual input function.""" + batch_size = params["batch_size"] + + num_examples = len(features) + + # This is for demo purposes and does NOT scale to large data sets. We do + # not use Dataset.from_generator() because that uses tf.py_func which is + # not TPU compatible. The right way to load data is with TFRecordReader. + d = tf.data.Dataset.from_tensor_slices({ + "input_ids": + tf.constant( + all_input_ids, shape=[num_examples, seq_length], + dtype=tf.int32), + "input_mask": + tf.constant( + all_input_mask, + shape=[num_examples, seq_length], + dtype=tf.int32), + "segment_ids": + tf.constant( + all_segment_ids, + shape=[num_examples, seq_length], + dtype=tf.int32), + "label_ids": + tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32), + }) + + if is_training: + d = d.repeat() + d = d.shuffle(buffer_size=100) + + d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder) + return d + + return input_fn + + +# This function is not used by this file but is still used by the Colab and +# people who depend on it. +def convert_examples_to_features(examples, label_list, max_seq_length, + tokenizer): + """Convert a set of `InputExample`s to a list of `InputFeatures`.""" + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) + + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + features.append(feature) + return features + + +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mrpc": MrpcProcessor, + "xnli": XnliProcessor, + } + + tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, + FLAGS.init_checkpoint) + + if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: + raise ValueError( + "At least one of `do_train`, `do_eval` or `do_predict' must be True.") + + bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) + + if FLAGS.max_seq_length > bert_config.max_position_embeddings: + raise ValueError( + "Cannot use sequence length %d because the BERT model " + "was only trained up to sequence length %d" % + (FLAGS.max_seq_length, bert_config.max_position_embeddings)) + + tf.gfile.MakeDirs(FLAGS.output_dir) + + task_name = FLAGS.task_name.lower() + + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name]() + + label_list = processor.get_labels() + + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + tpu_cluster_resolver = None + if FLAGS.use_tpu and FLAGS.tpu_name: + tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( + FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) + + is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 + run_config = tf.contrib.tpu.RunConfig( + cluster=tpu_cluster_resolver, + master=FLAGS.master, + model_dir=FLAGS.output_dir, + save_checkpoints_steps=FLAGS.save_checkpoints_steps, + tpu_config=tf.contrib.tpu.TPUConfig( + iterations_per_loop=FLAGS.iterations_per_loop, + num_shards=FLAGS.num_tpu_cores, + per_host_input_for_training=is_per_host)) + + train_examples = None + num_train_steps = None + num_warmup_steps = None + if FLAGS.do_train: + train_examples = processor.get_train_examples(FLAGS.data_dir) + num_train_steps = int( + len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) + num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) + + model_fn = model_fn_builder( + bert_config=bert_config, + num_labels=len(label_list), + init_checkpoint=FLAGS.init_checkpoint, + learning_rate=FLAGS.learning_rate, + num_train_steps=num_train_steps, + num_warmup_steps=num_warmup_steps, + use_tpu=FLAGS.use_tpu, + use_one_hot_embeddings=FLAGS.use_tpu) + + # If TPU is not available, this will fall back to normal Estimator on CPU + # or GPU. + estimator = tf.contrib.tpu.TPUEstimator( + use_tpu=FLAGS.use_tpu, + model_fn=model_fn, + config=run_config, + train_batch_size=FLAGS.train_batch_size, + eval_batch_size=FLAGS.eval_batch_size, + predict_batch_size=FLAGS.predict_batch_size) + + if FLAGS.do_train: + train_file = os.path.join(FLAGS.output_dir, "train.tf_record") + file_based_convert_examples_to_features( + train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) + tf.logging.info("***** Running training *****") + tf.logging.info(" Num examples = %d", len(train_examples)) + tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) + tf.logging.info(" Num steps = %d", num_train_steps) + train_input_fn = file_based_input_fn_builder( + input_file=train_file, + seq_length=FLAGS.max_seq_length, + is_training=True, + drop_remainder=True) + estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) + + if FLAGS.do_eval: + eval_examples = processor.get_dev_examples(FLAGS.data_dir) + num_actual_eval_examples = len(eval_examples) + if FLAGS.use_tpu: + # TPU requires a fixed batch size for all batches, therefore the number + # of examples must be a multiple of the batch size, or else examples + # will get dropped. So we pad with fake examples which are ignored + # later on. These do NOT count towards the metric (all tf.metrics + # support a per-instance weight, and these get a weight of 0.0). + while len(eval_examples) % FLAGS.eval_batch_size != 0: + eval_examples.append(PaddingInputExample()) + + eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") + file_based_convert_examples_to_features( + eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) + + tf.logging.info("***** Running evaluation *****") + tf.logging.info(" Num examples = %d (%d actual, %d padding)", + len(eval_examples), num_actual_eval_examples, + len(eval_examples) - num_actual_eval_examples) + tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) + + # This tells the estimator to run through the entire set. + eval_steps = None + # However, if running eval on the TPU, you will need to specify the + # number of steps. + if FLAGS.use_tpu: + assert len(eval_examples) % FLAGS.eval_batch_size == 0 + eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) + + eval_drop_remainder = True if FLAGS.use_tpu else False + eval_input_fn = file_based_input_fn_builder( + input_file=eval_file, + seq_length=FLAGS.max_seq_length, + is_training=False, + drop_remainder=eval_drop_remainder) + + result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) + + output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") + with tf.gfile.GFile(output_eval_file, "w") as writer: + tf.logging.info("***** Eval results *****") + for key in sorted(result.keys()): + tf.logging.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + if FLAGS.do_predict: + predict_examples = processor.get_test_examples(FLAGS.data_dir) + num_actual_predict_examples = len(predict_examples) + if FLAGS.use_tpu: + # TPU requires a fixed batch size for all batches, therefore the number + # of examples must be a multiple of the batch size, or else examples + # will get dropped. So we pad with fake examples which are ignored + # later on. + while len(predict_examples) % FLAGS.predict_batch_size != 0: + predict_examples.append(PaddingInputExample()) + + predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") + file_based_convert_examples_to_features(predict_examples, label_list, + FLAGS.max_seq_length, tokenizer, + predict_file) + + tf.logging.info("***** Running prediction*****") + tf.logging.info(" Num examples = %d (%d actual, %d padding)", + len(predict_examples), num_actual_predict_examples, + len(predict_examples) - num_actual_predict_examples) + tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) + + predict_drop_remainder = True if FLAGS.use_tpu else False + predict_input_fn = file_based_input_fn_builder( + input_file=predict_file, + seq_length=FLAGS.max_seq_length, + is_training=False, + drop_remainder=predict_drop_remainder) + + result = estimator.predict(input_fn=predict_input_fn) + + output_predict_file = os.path.join(FLAGS.output_dir, "test_results.tsv") + with tf.gfile.GFile(output_predict_file, "w") as writer: + num_written_lines = 0 + tf.logging.info("***** Predict results *****") + for (i, prediction) in enumerate(result): + probabilities = prediction["probabilities"] + if i >= num_actual_predict_examples: + break + output_line = "\t".join( + str(class_probability) + for class_probability in probabilities) + "\n" + writer.write(output_line) + num_written_lines += 1 + assert num_written_lines == num_actual_predict_examples + + +if __name__ == "__main__": + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("task_name") + flags.mark_flag_as_required("vocab_file") + flags.mark_flag_as_required("bert_config_file") + flags.mark_flag_as_required("output_dir") + tf.app.run() diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py new file mode 100644 index 0000000..5559533 --- /dev/null +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -0,0 +1,199 @@ +''' +Created on Jan 13, 2018 + +@author: manu +''' +import os +import glob +import logging + +#Import BERT related file - USED as is from Google/BERT +import modeling +import optimization +import run_classifier +import tokenization +import tensorflow as tf + +import settings +import datetime + +from app import app + +INCEPTIONV3_TOPLESS_MODEL_PATH = app.config['INCEPTIONV3_TOPLESS_MODEL_PATH'] +BERT_MODEL_PATH = app.config['BERT_MODEL_PATH'] + + +class BertTransferLeaner: + def __init__(self, model_name): + self.model_name = model_name + + def traineval_model(self, local_dir, + nb_epoch, + batch_size): + """ + Use the BERT Uncased language model to train on + new data + """ + tf.logging.set_verbosity(tf.logging.INFO) + logging.info("*:BERT MODEL PATH:%s",BERT_MODEL_PATH) + logging.info("*:Local Dir%s",local_dir) + + + mod_name = self.model_name + BERT_MODEL = 'uncased_L-12_H-768_A-12' + BERT_PRETRAINED_DIR = os.path.join(BERT_MODEL_PATH,'uncased_L-12_H-768_A-12') + OUTPUT_DIR = os.path.join(local_dir,'output_bert') + DATA_DIR = os.path.join(local_dir,'data') + logging.info('***** Model output directory: %s*****',OUTPUT_DIR) + logging.info('***** BERT pretrained directory: %s *****',BERT_PRETRAINED_DIR) + + TRAIN_BATCH_SIZE = 32 + EVAL_BATCH_SIZE = 8 + LEARNING_RATE = 2e-5 + NUM_TRAIN_EPOCHS = 3.0 + WARMUP_PROPORTION = 0.1 + MAX_SEQ_LENGTH = 128 + # Model configs + # if you wish to finetune a model on a larger dataset, use larger interval + SAVE_CHECKPOINTS_STEPS = 1000 + # each checpoint weights about 1,5gb + ITERATIONS_PER_LOOP = 1000 + NUM_TPU_CORES = 8 + + VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR,'vocab.txt') + BERT_CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR,'bert_config.json') + INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt') + DO_LOWER_CASE = BERT_MODEL.startswith('uncased') + + bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) + tf.gfile.MakeDirs(OUTPUT_DIR) + processor = run_classifier.ColaProcessor() + label_list = processor.get_labels() + tokenizer = tokenization.FullTokenizer( + vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) + + # Since training will happen on GPU, we won't need a cluster resolver + tpu_cluster_resolver = None + # TPUEstimator also supports training on CPU and GPU. You don't need to define a separate tf.estimator.Estimator. + run_config = tf.contrib.tpu.RunConfig( + cluster=tpu_cluster_resolver, + model_dir=OUTPUT_DIR, + save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS, + tpu_config=tf.contrib.tpu.TPUConfig( + iterations_per_loop=ITERATIONS_PER_LOOP, + num_shards=NUM_TPU_CORES, + per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2)) + + train_examples = None + num_train_steps = None + num_warmup_steps = None + train_examples = processor.get_train_examples(DATA_DIR) + num_train_steps = int( + len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS) + num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION) + + model_fn = run_classifier.model_fn_builder( + bert_config=bert_config, + num_labels=len(label_list), + init_checkpoint=INIT_CHECKPOINT, + learning_rate=LEARNING_RATE, + num_train_steps=num_train_steps, + num_warmup_steps=num_warmup_steps, + use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available + use_one_hot_embeddings=False) #Try with True + + estimator = tf.contrib.tpu.TPUEstimator( + use_tpu=False, # If False training will fall on CPU or GPU, depending on what is available + model_fn=model_fn, + config=run_config, + train_batch_size=TRAIN_BATCH_SIZE, + eval_batch_size=EVAL_BATCH_SIZE) + + # Train the model. + logging.info('Starting Training...') + train_file = os.path.join(OUTPUT_DIR, "train.tf_record") + run_classifier.file_based_convert_examples_to_features( + train_examples, label_list, MAX_SEQ_LENGTH, tokenizer, train_file) + tf.logging.info('***** Started training at {} *****'.format(datetime.datetime.now())) + tf.logging.info(' Num examples = {}'.format(len(train_examples))) + tf.logging.info(' Batch size = {}'.format(TRAIN_BATCH_SIZE)) + tf.logging.info(" Num steps = %d", num_train_steps) + train_input_fn = run_classifier.file_based_input_fn_builder( + input_file=train_file, + seq_length=MAX_SEQ_LENGTH, + is_training=True, + drop_remainder=True) + estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) + print('***** Finished training at {} *****'.format(datetime.datetime.now())) + # Do Eval + logging.info('Starting Eval..') + eval_examples = processor.get_dev_examples(DATA_DIR) + num_actual_eval_examples = len(eval_examples) + eval_file = os.path.join(OUTPUT_DIR, "eval.tf_record") + file_based_convert_examples_to_features( + eval_examples, label_list, MAX_SEQ_LENGTH, tokenizer, eval_file) + + tf.logging.info("***** Running evaluation *****") + tf.logging.info(" Num examples = %d (%d actual, %d padding)", + len(eval_examples), num_actual_eval_examples, + len(eval_examples) - num_actual_eval_examples) + tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) + eval_steps = None + + eval_input_fn = file_based_input_fn_builder( + input_file=eval_file, + seq_length=FLAGS.max_seq_length, + is_training=False, + drop_remainder=False) + + result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) + + output_eval_file = os.path.join(OUTPUT_DIR, "eval_results.txt") + with tf.gfile.GFile(output_eval_file, "w") as writer: + tf.logging.info("***** Eval results *****") + for key in sorted(result.keys()): + tf.logging.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return result + + def __setup_to_finetune(self, model, nb_layer_to_freeze): + """ + Freeze the bottom NB_IV3_LAYERS and retrain the remaining top layers. + note: NB_IV3_LAYERS corresponds to the top 2 inception blocks in the inceptionv3 arch + Args: + model: keras model + """ + for layer in model.layers[:nb_layer_to_freeze]: + layer.trainable = False + for layer in model.layers[nb_layer_to_freeze:]: + layer.trainable = True + model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy']) + + def __setup_to_transfer_learn(self, model, base_model): + """Freeze all layers and compile the model""" + for layer in base_model.layers: + layer.trainable = False + model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) + + def __add_new_last_layer(self, topless_model, nb_classes): + """ + add the last layer to the topless model + """ + x = topless_model.output + x = GlobalAveragePooling2D()(x) + x = Dense(settings.FC_SIZE, activation='relu')(x) #new FC layer, random init + predictions = Dense(nb_classes, activation='softmax')(x) #new softmax layer + model = Model(input=topless_model.input, output=predictions) + return model + + def __get_nb_files(self, directory): + """Get number of files by searching local dir recursively""" + + if not os.path.exists(directory): + return 0 + cnt = 0 + for r, dirs, files in os.walk(directory): + for dr in dirs: + cnt += len(glob.glob(os.path.join(r, dr + "/*"))) + return cnt diff --git a/src/app/models/SentimentV1/sentiment_infer_server.py b/src/app/models/SentimentV1/sentiment_infer_server.py new file mode 100644 index 0000000..e2cb3bd --- /dev/null +++ b/src/app/models/SentimentV1/sentiment_infer_server.py @@ -0,0 +1,75 @@ +''' +Created on Jan 22, 2019 + +@author: manu +''' +import os +import redis +import time +import json +from textblob import TextBlob +import logging +from collections import defaultdict + + +#helpers +import settings + +pool = redis.ConnectionPool(host='redis', port=6379, db=0) +db = redis.Redis(connection_pool=pool) + +class sentimentV1_inference_server: + def __init__(self): + # pre-load some models here on start + self.loaded_models = {} + + def run_sentimentV1_infernece_server(self): + ''' + run the inference server for Sentiment Analysis + + Pull sentence from the Redis, predict + return the response to the redis + + Sentecnes are tracked using is their id + ''' + logging.info("Sentiment Inference Server running") + while True: + queue = db.lrange(settings.TEXT_QUEUE, 0, settings.BATCH_SIZE) #Is this queue different from the Queue in API path + textIDs = defaultdict(list) #dict to hold sentence and id for a model type + text_list=[] + sent_list = [] + num_text=0 + for q in queue: + q = json.loads(q.decode("utf-8")) + + model_name = str(q['model_name']) + id = q['id'] + sentence = q['text'] + logging.info("Sentence in server:%s", sentence) + + text_list.append({"model_name":model_name,"id":id, "text":sentence}) + textIDs[model_name].append(q['id']) + num_text += 1 + + sent_list.append(sentence) + + + if textIDs: + logging.info("* Predicting for {} of Models".format(len(textIDs.keys()))) + logging.info("* Number of Sentences: {}".format(num_text)) + + + for t in text_list: + logging.info("Text is:%s",t["text"]) + preds = TextBlob(t["text"]) + res = {"polarity":preds.sentiment.polarity,"subjectvity":preds.sentiment.subjectivity} + db.set(t["id"], json.dumps(res)) + + db.ltrim(settings.TEXT_QUEUE, len(textIDs), -1) + + # sleep and wait + time.sleep(settings.SERVER_SLEEP) + +if __name__ == "__main__": + this_server = sentimentV1_inference_server() + this_server.run_sentimentV1_infernece_server() diff --git a/src/app/models/SentimentV1/settings.py b/src/app/models/SentimentV1/settings.py new file mode 100644 index 0000000..c26bdd9 --- /dev/null +++ b/src/app/models/SentimentV1/settings.py @@ -0,0 +1,6 @@ +import os + +BATCH_SIZE = 2 +SERVER_SLEEP = 0.5 +NUM_LABEL_TO_RETURN = 2 +TEXT_QUEUE = 'sentimentV1_text_queue' diff --git a/src/app/models/SentimentV1/tokenization.py b/src/app/models/SentimentV1/tokenization.py new file mode 100644 index 0000000..8a40a0f --- /dev/null +++ b/src/app/models/SentimentV1/tokenization.py @@ -0,0 +1,399 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +#from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import unicodedata +import six +import tensorflow as tf + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % (actual_flag, init_checkpoint, + model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with tf.gfile.GFile(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/src/app/models/__init__.py b/src/app/models/__init__.py index fa12c8f..cf1a0af 100644 --- a/src/app/models/__init__.py +++ b/src/app/models/__init__.py @@ -1,2 +1,3 @@ from .mnist import * -from .InceptionV3 import * \ No newline at end of file +from .InceptionV3 import * +from .SentimentV1 import * diff --git a/src/app/tasks.py b/src/app/tasks.py index 009fd06..d3323f3 100644 --- a/src/app/tasks.py +++ b/src/app/tasks.py @@ -12,13 +12,18 @@ from app import app from .apis.InceptionV3 import API_helpers +from .apis.SentimentV1 import API_helpers_nlp from .models.InceptionV3 import inceptionV3_transfer_retraining +from .models.SentimentV1 import sentimentV1_transfer_retraining CLIENT_SLEEP = app.config['CLIENT_SLEEP'] INV3_TRANSFER_NB_EPOCH = app.config['INV3_TRANSFER_NB_EPOCH'] INV3_TRANSFER_BATCH_SIZE = app.config['INV3_TRANSFER_BATCH_SIZE'] INCEPTIONV3_IMAGE_QUEUE = app.config['INCEPTIONV3_IMAGE_QUEUE'] INCEPTIONV3_TOPLESS_MODEL_PATH = app.config['INCEPTIONV3_TOPLESS_MODEL_PATH'] + +SENTIMENT_TEXT_QUEUE = app.config['SENTIMENT_TEXT_QUEUE'] + TEMP_FOLDER = os.path.join('./tmp') @michaniki_celery_app.task() @@ -31,31 +36,31 @@ def async_retrain(model_name, """ retrain model resume training - """ - # download image data to local + """ + # download image data to local image_data_path = API_helpers.download_a_dir_from_s3(s3_bucket_name, - s3_bucket_prefix, + s3_bucket_prefix, local_path = TEMP_FOLDER) try: this_model_path = os.path.join("app", "models", "InceptionV3", model_name, model_name + ".h5") # load the model this_model = load_model(this_model_path) - + this_retrainer = inceptionV3_transfer_retraining.InceptionRetrainer(model_name) - + # return the retraiend new model new_model, history = this_retrainer.retrain(this_model, - image_data_path, - nb_epoch, + image_data_path, + nb_epoch, batch_size) - + print "* Celery Transfer: Retrained Model Saved at: {}".format(this_model_path) # replace the current model new_model.save(this_model_path) - + # remove the local image path shutil.rmtree(image_data_path, ignore_errors=True) - + final_trn_acc = history.history['acc'][-1] final_val_acc = history.history['val_acc'][-1] return final_trn_acc, final_val_acc @@ -63,7 +68,7 @@ def async_retrain(model_name, # remove the local image path shutil.rmtree(image_data_path, ignore_errors=True) raise - + @michaniki_celery_app.task() def async_transfer(model_name, s3_bucket_name, @@ -76,36 +81,59 @@ def async_transfer(model_name, new_model_folder_path = os.path.join("app", "models", "InceptionV3", model_name) if not os.path.exists(new_model_folder_path): os.makedirs(new_model_folder_path) - + + print "TEMP_FOLDER->{}".format(TEMP_FOLDER) + image_data_path = API_helpers.download_a_dir_from_s3(bucket_name = s3_bucket_name, bucket_prefix = s3_bucket_prefix, local_path = TEMP_FOLDER) try: # init the transfer learning manager this_IV3_transfer = inceptionV3_transfer_retraining.InceptionTransferLeaner(model_name) - new_model, label_dict, history = this_IV3_transfer.transfer_model(image_data_path, + + + new_model, label_dict, history = this_IV3_transfer.transfer_model(image_data_path, nb_epoch = INV3_TRANSFER_NB_EPOCH, batch_size = INV3_TRANSFER_BATCH_SIZE) - + # save the model .h5 file and the class label file new_model_path = os.path.join(new_model_folder_path, model_name + ".h5") new_label_path = os.path.join(new_model_folder_path, model_name + ".json") new_model.save(new_model_path) API_helpers.save_classes_label_dict(label_dict, new_label_path) print "* Celery Transfer: New Model Saved at: {}".format(new_model_path) - + # delete the image folder here: shutil.rmtree(image_data_path, ignore_errors=True) - + # return the train and val acc: final_trn_acc = history.history['acc'][-1] final_val_acc = history.history['val_acc'][-1] return final_trn_acc, final_val_acc - + except Exception as err: # catch any error shutil.rmtree(new_model_folder_path, ignore_errors=True) shutil.rmtree(image_data_path, ignore_errors=True) raise - - \ No newline at end of file + +@michaniki_celery_app.task() +def async_berttrain(model_name, + s3_bucket_name, + s3_bucket_prefix, + id): + """ + train a model using BERT pre-trained model + """ + text_data_path = API_helpers_nlp.download_a_dir_from_s3(s3_bucket_name, + s3_bucket_prefix, + local_path = TEMP_FOLDER) + + logging.info('*Text Data Path:%s',text_data_path) + try: + bert_transfer = sentimentV1_transfer_retraining.BertTransferLeaner(model_name) + new_model_eval_res = bert_transfer.traineval_model(text_data_path) + return new_model_eval_res + except Exception as err: + shutil.rmtree(text_data_path, ignore_errors=True) + raise diff --git a/src/config.py b/src/config.py index 5fcf0de..c37dfdd 100644 --- a/src/config.py +++ b/src/config.py @@ -7,10 +7,12 @@ CLIENT_SLEEP = env.str('CLIENT_SLEEP', default=0.5) # settings for InceptionV3 -INCEPTIONV3_TOPLESS_MODEL_PATH = env.str('INCEPTIONV3_TOPLESS_MODEL_PATH', default=os.path.join("app", "models", "InceptionV3", "topless",'topless.h5')) +INCEPTIONV3_TOPLESS_MODEL_PATH = env.str('INCEPTIONV3_TOPLESS_MODEL_PATH', default=os.path.join("app", "models", "InceptionV3", "topless", 'topless.h5')) INCEPTIONV3_IMAGE_QUEUE = env.str('INCEPTIONV3_IMAGE_QUEUE', default='inceptionV3_image_queue') INV3_TRANSFER_NB_EPOCH = env.str('INV3_TRANSFER_NB_EPOCH', default=3) INV3_TRANSFER_BATCH_SIZE = env.str('INV3_TRANSFER_BATCH_SIZE', default=2) +SENTIMENT_TEXT_QUEUE = env.str('SENTIMENT_TEXT_QUEUE', default='sentimentV1_text_queue') +BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12") # setting for mysql db # parsed from environment variables @@ -22,4 +24,4 @@ # redis url for celery BROKER_URL = env.str('BROKER_URL', default='redis://redis:6379/0') -BACKEND_URL = env.str('BACKEND_URL', default='redis://redis:6379/0') \ No newline at end of file +BACKEND_URL = env.str('BACKEND_URL', default='redis://redis:6379/0') diff --git a/src/install_base_model.py b/src/install_base_model.py index c61b3e6..031c6bb 100644 --- a/src/install_base_model.py +++ b/src/install_base_model.py @@ -1,9 +1,16 @@ import os import redis +import logging +import requests +import zipfile, io +from tqdm import tqdm +import math from keras.applications.inception_v3 import InceptionV3 BASE_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "base", "base.h5") TOPLESS_MODEL_PATH = os.path.join("app", "models", "InceptionV3", "topless") +BERT_MODEL_PATH = os.path.join("app", "models", "SentimentV1", "uncased_L-12_H-768_A-12","bert_model.ckpt.data-00000-of-00001") +BERT_DIR_PATH = os.path.join("app", "models", "SentimentV1") # loading base model if os.path.exists(BASE_MODEL_PATH): @@ -11,10 +18,10 @@ else: print "* Starting: No Base Model Found. Loading..." base_model = InceptionV3(include_top=True, weights='imagenet',input_shape=(299, 299, 3)) - + base_model.save(BASE_MODEL_PATH) print "* Starting: Base Model Saved!" - + # loading topless model if os.path.exists(TOPLESS_MODEL_PATH): print "* Starting: Found Topless Model." @@ -24,9 +31,26 @@ base_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(299, 299, 3)) base_model.save(os.path.join(TOPLESS_MODEL_PATH, "topless.h5")) print "* Starting: Topless Model Saved!" - + +if os.path.exists(BERT_MODEL_PATH): + logging.info("* Found BERT uncased model") +else: + logging.info("BERT model not found. Downloading....") + BERT_UNCASED_URL = 'https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip' + r = requests.get(BERT_UNCASED_URL, stream=True) + total_size = int(r.headers.get('content-length', 0)); + block_size = 1024 + wrote=0 + with open(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip'), 'wb') as f: + for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size//block_size) , unit='KB', unit_scale=True): + wrote = wrote + len(data) + f.write(data) + + z = zipfile.ZipFile(os.path.join(BERT_DIR_PATH,'uncased_L-12_H-768_A-12.zip')) + z.extractall(BERT_DIR_PATH) + # clean up the died images upon start: # need to wait a bit for redis container to start up pool = redis.ConnectionPool(host='redis', port=6379, db=0) db = redis.Redis(connection_pool=pool) -db.flushall() \ No newline at end of file +db.flushall() diff --git a/src/requirements.txt b/src/requirements.txt index db9ffb5..48ec5e8 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -90,8 +90,10 @@ tensorboard==1.8.0 tensorflow==1.5.0 termcolor==1.1.0 tornado==5.0.2 +tqdm==4.28.0 traitlets==4.3.2 urllib3==1.23 uWSGI==2.0.17 wcwidth==0.1.7 Werkzeug==0.14.1 +textblob==0.15.0 diff --git a/src/requirementssenti.txt b/src/requirementssenti.txt new file mode 100644 index 0000000..a889dbe --- /dev/null +++ b/src/requirementssenti.txt @@ -0,0 +1,154 @@ +absl-py==0.6.1 +argh==0.26.2 +asn1crypto==0.24.0 +astor==0.7.1 +backcall==0.1.0 +backports-abc==0.5 +base58==1.0.3 +beautifulsoup4==4.6.3 +bleach==3.0.2 +boto==2.49.0 +boto3==1.9.78 +botocore==1.12.78 +Bottleneck==1.2.1 +bz2file==0.98 +certifi==2018.11.29 +cffi==1.11.5 +chardet==3.0.4 +Click==7.0 +cryptography==2.4.2 +cycler==0.10.0 +cymem==2.0.2 +cytoolz==0.9.0.1 +dataclasses==0.6 +decorator==4.3.0 +Deprecated==1.2.4 +dill==0.2.8.2 +docutils==0.14 +entrypoints==0.2.3 +enum-compat==0.0.2 +fastai==1.0.40 +fastprogress==0.1.18 +flair==0.4.0 +Flask==1.0.2 +future==0.17.1 +future-fstrings==0.4.5 +futures==3.1.1 +gast==0.2.2 +gensim==3.4.0 +gluonnlp==0.5.0.post0 +graphviz==0.8.4 +grpcio==1.17.1 +h5py==2.9.0 +hyperopt==0.1.1 +idna==2.8 +ipykernel==5.1.0 +ipython==7.2.0 +ipython-genutils==0.2.0 +ipywidgets==7.4.2 +itsdangerous==1.1.0 +jedi==0.13.2 +Jinja2==2.10 +jmespath==0.9.3 +jsonschema==2.6.0 +jupyter==1.0.0 +jupyter-client==5.2.4 +jupyter-console==6.0.0 +jupyter-core==4.4.0 +Keras==2.2.4 +Keras-Applications==1.0.6 +Keras-Preprocessing==1.0.5 +kiwisolver==1.0.1 +Markdown==3.0.1 +MarkupSafe==1.1.0 +matplotlib==3.0.0 +mistune==0.8.4 +mkl-fft==1.0.6 +mkl-random==1.0.1 +mock==2.0.0 +mpld3==0.3 +msgpack==0.5.6 +msgpack-numpy==0.4.3.2 +murmurhash==1.0.1 +mxnet-cu90==1.3.1 +mypy==0.650 +mypy-extensions==0.4.1 +networkx==2.2 +nltk==3.4 +notebook==5.7.4 +numexpr==2.6.8 +numpy==1.14.6 +olefile==0.46 +packaging==18.0 +pandas==0.23.4 +pandocfilters==1.4.2 +parso==0.3.1 +pathtools==0.1.2 +pbr==5.1.1 +pexpect==4.6.0 +pickleshare==0.7.5 +Pillow==5.4.1 +plac==0.9.6 +preshed==2.0.1 +prometheus-client==0.5.0 +prompt-toolkit==2.0.7 +protobuf==3.6.1 +psutil==5.4.8 +ptyprocess==0.6.0 +pycparser==2.19 +Pygments==2.3.1 +pymongo==3.7.2 +pyOpenSSL==18.0.0 +pyparsing==2.3.0 +PySocks==1.6.8 +python-dateutil==2.7.5 +pytorch-pretrained-bert==0.3.0 +pytz==2018.7 +PyYAML==3.13 +pyzmq==17.1.2 +qtconsole==4.4.3 +redis==3.0.1 +regex==2018.1.10 +requests==2.21.0 +s3transfer==0.1.13 +scikit-learn==0.20.1 +scipy==1.1.0 +seaborn==0.9.0 +segtok==1.5.7 +Send2Trash==1.5.0 +singledispatch==3.4.0.3 +six==1.12.0 +sklearn==0.0 +smart-open==1.8.0 +spacy==2.0.18 +sqlitedict==1.6.0 +streamlit==0.23.0 +tensorboard==1.12.2 +tensorflow==1.12.0 +tensorflow-hub==0.2.0 +termcolor==1.1.0 +terminado==0.8.1 +testpath==0.4.2 +textblob==0.15.2 +thinc==6.12.1 +tokenize-rt==2.1.0 +toml==0.10.0 +toolz==0.9.0 +torch==1.0.0 +torchtext==0.3.1 +torchvision==0.2.1 +tornado==5.1.1 +tqdm==4.26.0 +traitlets==4.3.2 +typed-ast==1.1.1 +typing==3.6.4 +tzlocal==1.5.1 +ujson==1.35 +urllib3==1.24.1 +watchdog==0.9.0 +wcwidth==0.1.7 +webencodings==0.5.1 +Werkzeug==0.14.1 +widgetsnbextension==3.4.2 +wrapt==1.10.11 +xgboost==0.81