From dcd77d774ed675d0638907028862d30bd1199be9 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 6 Feb 2019 17:12:05 -0800 Subject: [PATCH 01/10] trying new feature --- .../sentimentV1_transfer_retraining.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 4bdbc96..c8935ca 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -15,6 +15,7 @@ import run_classifier import tokenization import tensorflow as tf +import csv import settings import datetime @@ -28,6 +29,17 @@ class BertTransferLeaner: def __init__(self, model_name): self.model_name = model_name + + def getlabel(self,data_dir): + with tf.gfile.Open(os.path.join(data_dir, "train.tsv"),"r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=None) + label = [] + for line in reader: + line_label = line[1] + if line_label not in label: + label.append(line_label) + return label + def traineval_model(self, local_dir, nb_epoch, @@ -71,7 +83,9 @@ def traineval_model(self, local_dir, bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) tf.gfile.MakeDirs(OUTPUT_DIR) processor = run_classifier.ColaProcessor() - label_list = processor.get_labels() + #label_list = processor.get_labels() + label_list = self.getlabel(DATA_DIR) + print label_list tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) @@ -215,7 +229,8 @@ def test_model(self, local_dir, bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) tf.gfile.MakeDirs(OUTPUT_DIR) processor = run_classifier.ColaProcessor() - label_list = processor.get_labels() + #label_list = processor.get_labels() + label_list = self.getlabel() tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) From f05c094664c2a6e9b6eb6ac5d1aba35ef61b854b Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 6 Feb 2019 22:52:32 -0800 Subject: [PATCH 02/10] fix indent --- .../SentimentV1/sentimentV1_transfer_retraining.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index c8935ca..07b2170 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -32,12 +32,12 @@ def __init__(self, model_name): def getlabel(self,data_dir): with tf.gfile.Open(os.path.join(data_dir, "train.tsv"),"r") as f: - reader = csv.reader(f, delimiter="\t", quotechar=None) - label = [] - for line in reader: - line_label = line[1] - if line_label not in label: - label.append(line_label) + reader = csv.reader(f, delimiter="\t", quotechar=None) + label = [] + for line in reader: + line_label = line[1] + if line_label not in label: + label.append(line_label) return label From cc9a838d5e2f733331335ad0d9bf629af1e07810 Mon Sep 17 00:00:00 2001 From: Manu Date: Thu, 7 Feb 2019 09:43:09 -0800 Subject: [PATCH 03/10] fix eval label --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 07b2170..a25a3e1 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -152,6 +152,8 @@ def traineval_model(self, local_dir, # Do Eval logging.info('Starting Eval..') eval_examples = processor.get_dev_examples(DATA_DIR) + label_list = self.getlabel(DATA_DIR) + print label_list num_actual_eval_examples = len(eval_examples) eval_file = os.path.join(OUTPUT_DIR, "eval.tf_record") run_classifier.file_based_convert_examples_to_features( From 236dcb0a4f383adf83c76c65fc733a21621bc5e8 Mon Sep 17 00:00:00 2001 From: Manu Date: Thu, 7 Feb 2019 11:28:46 -0800 Subject: [PATCH 04/10] last commit not needed, so removing those changes --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index a25a3e1..07b2170 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -152,8 +152,6 @@ def traineval_model(self, local_dir, # Do Eval logging.info('Starting Eval..') eval_examples = processor.get_dev_examples(DATA_DIR) - label_list = self.getlabel(DATA_DIR) - print label_list num_actual_eval_examples = len(eval_examples) eval_file = os.path.join(OUTPUT_DIR, "eval.tf_record") run_classifier.file_based_convert_examples_to_features( From 23ec166a0eb471d67776550fe61a46d40d0c33ff Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 8 Feb 2019 22:57:28 -0800 Subject: [PATCH 05/10] trying celery task --- docker-compose-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml index bab364c..40cc513 100644 --- a/docker-compose-gpu.yml +++ b/docker-compose-gpu.yml @@ -71,7 +71,7 @@ services: context: ./src dockerfile: Dockerfile-gpu runtime: nvidia - command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] + command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info','--max-tasks-per-child','1'] volumes: - ./src:/opt/src networks: From 46d1f1a05048477ce46ba18772bdec100c08852f Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 9 Feb 2019 01:07:23 -0800 Subject: [PATCH 06/10] download file fix --- src/app/apis/SentimentV1/API_helpers_nlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py index f088cb3..8cf1c5e 100644 --- a/src/app/apis/SentimentV1/API_helpers_nlp.py +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -64,7 +64,7 @@ def download_a_dir_from_s3(bucket_name, local_path): logging.info("* Helper: Text Loaded at: {}".format(output_path)) return output_path -def download_test_file_from_s3(bucket_name, bucket_prefix, local_path): +def download_test_file_from_s3(bucket_name, local_path): """ download the folder from S3 From 0049fc9d43d3357ccb353b3cea1f3a83e8ded6b2 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 9 Feb 2019 01:13:10 -0800 Subject: [PATCH 07/10] fix label issue --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 07b2170..b866ea4 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -230,7 +230,7 @@ def test_model(self, local_dir, tf.gfile.MakeDirs(OUTPUT_DIR) processor = run_classifier.ColaProcessor() #label_list = processor.get_labels() - label_list = self.getlabel() + label_list = self.getlabel(DATA_DIR) tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) From 2458e885661bccd6391291c1f106750217a06040 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 9 Feb 2019 01:20:35 -0800 Subject: [PATCH 08/10] add print statement --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index b866ea4..34cece1 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -231,6 +231,7 @@ def test_model(self, local_dir, processor = run_classifier.ColaProcessor() #label_list = processor.get_labels() label_list = self.getlabel(DATA_DIR) + print label_list tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) From 8eae9f520bfb383e68e49f677c74b14eed062754 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 9 Feb 2019 01:45:05 -0800 Subject: [PATCH 09/10] fix test issue --- .../sentimentV1_transfer_retraining.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 34cece1..4b7a640 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -39,6 +39,28 @@ def getlabel(self,data_dir): if line_label not in label: label.append(line_label) return label + + def cal_test_examples(self,data_dir,firstlabel,set_type): + examples = [] + with tf.gfile.Open(os.path.join(data_dir, "test.tsv"), "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=None) + lines = [] + for line in reader: + lines.append(line) + for (i, line) in enumerate(lines): + # Only the test set has a header + if set_type == "test" and i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + label = firstlabel + else: + text_a = tokenization.convert_to_unicode(line[3]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples def traineval_model(self, local_dir, @@ -270,7 +292,8 @@ def test_model(self, local_dir, eval_batch_size=EVAL_BATCH_SIZE, predict_batch_size=PREDICT_BATCH_SIZE) - predict_examples = processor.get_test_examples(DATA_DIR) + #predict_examples = processor.get_test_examples(DATA_DIR) + predict_examples = self.cal_test_examples(DATA_DIR,label_list[0],"test") num_actual_predict_examples = len(predict_examples) predict_file = os.path.join(OUTPUT_DIR, "predict.tf_record") run_classifier.file_based_convert_examples_to_features(predict_examples, label_list, From 8b6e89bc5460fbfbe7c0385ecd0760181b6991c7 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 9 Feb 2019 01:49:11 -0800 Subject: [PATCH 10/10] fix test bug --- src/app/models/SentimentV1/sentimentV1_transfer_retraining.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 4b7a640..e4271bb 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -59,7 +59,7 @@ def cal_test_examples(self,data_dir,firstlabel,set_type): text_a = tokenization.convert_to_unicode(line[3]) label = tokenization.convert_to_unicode(line[1]) examples.append( - InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) return examples