diff --git a/docker-compose-gpu.yml b/docker-compose-gpu.yml index bab364c..40cc513 100644 --- a/docker-compose-gpu.yml +++ b/docker-compose-gpu.yml @@ -71,7 +71,7 @@ services: context: ./src dockerfile: Dockerfile-gpu runtime: nvidia - command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info'] + command: ['celery', '-A', 'app.celeryapp:michaniki_celery_app', 'worker', '-l', 'info','--max-tasks-per-child','1'] volumes: - ./src:/opt/src networks: diff --git a/src/app/apis/SentimentV1/API_helpers_nlp.py b/src/app/apis/SentimentV1/API_helpers_nlp.py index f088cb3..8cf1c5e 100644 --- a/src/app/apis/SentimentV1/API_helpers_nlp.py +++ b/src/app/apis/SentimentV1/API_helpers_nlp.py @@ -64,7 +64,7 @@ def download_a_dir_from_s3(bucket_name, local_path): logging.info("* Helper: Text Loaded at: {}".format(output_path)) return output_path -def download_test_file_from_s3(bucket_name, bucket_prefix, local_path): +def download_test_file_from_s3(bucket_name, local_path): """ download the folder from S3 diff --git a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py index 4bdbc96..e4271bb 100644 --- a/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py +++ b/src/app/models/SentimentV1/sentimentV1_transfer_retraining.py @@ -15,6 +15,7 @@ import run_classifier import tokenization import tensorflow as tf +import csv import settings import datetime @@ -28,6 +29,39 @@ class BertTransferLeaner: def __init__(self, model_name): self.model_name = model_name + + def getlabel(self,data_dir): + with tf.gfile.Open(os.path.join(data_dir, "train.tsv"),"r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=None) + label = [] + for line in reader: + line_label = line[1] + if line_label not in label: + label.append(line_label) + return label + + def cal_test_examples(self,data_dir,firstlabel,set_type): + examples = [] + with tf.gfile.Open(os.path.join(data_dir, "test.tsv"), "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=None) + lines = [] + for line in reader: + lines.append(line) + for (i, line) in enumerate(lines): + # Only the test set has a header + if set_type == "test" and i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + label = firstlabel + else: + text_a = tokenization.convert_to_unicode(line[3]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + def traineval_model(self, local_dir, nb_epoch, @@ -71,7 +105,9 @@ def traineval_model(self, local_dir, bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) tf.gfile.MakeDirs(OUTPUT_DIR) processor = run_classifier.ColaProcessor() - label_list = processor.get_labels() + #label_list = processor.get_labels() + label_list = self.getlabel(DATA_DIR) + print label_list tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) @@ -215,7 +251,9 @@ def test_model(self, local_dir, bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG_FILE) tf.gfile.MakeDirs(OUTPUT_DIR) processor = run_classifier.ColaProcessor() - label_list = processor.get_labels() + #label_list = processor.get_labels() + label_list = self.getlabel(DATA_DIR) + print label_list tokenizer = tokenization.FullTokenizer( vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE) @@ -254,7 +292,8 @@ def test_model(self, local_dir, eval_batch_size=EVAL_BATCH_SIZE, predict_batch_size=PREDICT_BATCH_SIZE) - predict_examples = processor.get_test_examples(DATA_DIR) + #predict_examples = processor.get_test_examples(DATA_DIR) + predict_examples = self.cal_test_examples(DATA_DIR,label_list[0],"test") num_actual_predict_examples = len(predict_examples) predict_file = os.path.join(OUTPUT_DIR, "predict.tf_record") run_classifier.file_based_convert_examples_to_features(predict_examples, label_list,