{\rtf1\ansi\ansicpg1252\cocoartf1561\cocoasubrtf600
{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
{\colortbl;\red255\green255\blue255;}
{\*\expandedcolortbl;;}
\margl1440\margr1440\vieww28240\viewh15160\viewkind0
\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0

\f0\fs24 \cf0 import numpy as np\
import pandas as pd\
import os\
import sys\
import random\
import keras\
import tensorflow as tf\
import json\
sys.path.insert(0, \'91gs://bertnglove/bert-master/')\
!cp -r 'gs://bertnglove/keras_bert' '/kaggle/working'\
BERT_PRETRAINED_DIR = 'gs://bertnglove/bert_uncased\'92\
print('***** BERT pretrained directory: \{\} *****'.format(BERT_PRETRAINED_DIR))\
import tokenization  #Actually keras_bert contains tokenization part, here just for convenience\
\
from keras_bert.keras_bert.bert import get_model\
from keras_bert.keras_bert.loader import load_trained_model_from_checkpoint\
from keras.optimizers import Adam\
adam = Adam(lr=2e-5,decay=0.01)\
maxlen = 50\
print('begin_build')\
config_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')\
checkpoint_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')\
model = load_trained_model_from_checkpoint(config_file, checkpoint_file, training=True,seq_len=maxlen)\
for i in range(110):\
    model.layers[i].trainable = False\
\
import logging\
import datetime\
import warnings\
import pickle\
\
from keras.engine.topology import Layer\
from keras import initializers, regularizers, constraints, optimizers, layers\
\
from keras.models import Model\
from sklearn.metrics import roc_auc_score\
from sklearn.model_selection import KFold\
\
from keras.layers import Dense,Input,Flatten,concatenate,Dropout,Lambda\
from keras.layers import Embedding, SpatialDropout1D, Dropout, add\
from keras.layers import CuDNNLSTM, GRU, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D\
import keras.backend as K\
import re\
import codecs\
\
import zipfile\
\
def build_marriage_model(embedding_matrix, model,num_aux_targets):\
    LSTM_UNITS = 128\
    DENSE_HIDDEN_UNITS = 512\
    words = Input(shape=(maxlen,))\
    glove_embs = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False)(words)\
    sequence_output  = model.layers[-7].output\
    conc = concatenate([sequence_output,glove_embs,])\
    x = Bidirectional(GRU(LSTM_UNITS, return_sequences=True))(conc)\
    x = Bidirectional(GRU(LSTM_UNITS, return_sequences=True))(x)\
    hidden = GlobalAveragePooling1D()(x)\
    hidden = Dense(DENSE_HIDDEN_UNITS, activation='relu',kernel_initializer=keras.initializers.TruncatedNormal(stddev=0.02))(hidden)\
    hidden = Dense(DENSE_HIDDEN_UNITS, activation='relu',kernel_initializer=keras.initializers.TruncatedNormal(stddev=0.02))(hidden)\
    pool_output = Dense(1, activation='sigmoid',kernel_initializer=keras.initializers.TruncatedNormal(stddev=0.02),name = 'real_output')(hidden)\
    aux_result = Dense(num_aux_targets, activation='sigmoid')(hidden)\
    print("mode inputs ", type(model.input))\
    model_inputs = [model.input[0],model.input[1],model.input[2],words]\
    model3  = Model(inputs=model_inputs, outputs=[pool_output,aux_result])\
    model3.compile(loss='binary_crossentropy', optimizer=adam)\
    model3.summary()\
    print(model_inputs)\
    return model3\
\
def get_coefs(word, *arr):\
    return word, np.asarray(arr, dtype='float32')\
\
def load_embeddings(path):\
    with open(path) as f:\
        return dict(get_coefs(*line.strip().split(' ')) for line in f)\
\
path = 'gs://bertnglove/glove300d.zip\'92\
\
with zipfile.ZipFile(path, \'93r\'94) as zip_ref:\
    zip_ref.extractall()\
\
embedding_file = \'93glove300d/glove.840B.300d.txt\'94\
\
embedding_index = load_embeddings(embedding_file)\
\
def convert_lines(example, maxlen,tokenizer):\
    max_seq_length = maxlen - 2\
    all_tokens = []\
    longer = 0\
    glove_lines = []\
    for i in range(example.shape[0]):\
        tokens_a = tokenizer.tokenize(example[i])\
        if len(tokens_a)>max_seq_length: # to account for CLS and SEP minus 2\
            tokens_a = tokens_a[:max_seq_length]\
            longer += 1\
        glove_lines.append(["[CLS]"]+tokens_a+["[SEP]"])\
        one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"]) + [0] * (max_seq_length - len(tokens_a))\
        assert len(one_token) == maxlen\
        all_tokens.append(one_token)\
    #print(longer)\
    return np.array(all_tokens), glove_lines\
    \
nb_epochs=1\
bsz = 32\
dict_path = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')\
tokenizer = tokenization.FullTokenizer(vocab_file=dict_path, do_lower_case=True)\
print('build tokenizer done')\
\
def tokenize_glove_lines(glove_lines, embedding_index):\
    token_index = 1 #index 0 reserved for padding\
    embedding_matrix = [np.zeros(300)] #first item in matrix for padding\
    token_to_word = \{\}\
    word_to_token = \{\}\
    token_to_word[0] = "<PAD>"\
    word_to_token["<PAD>"] = 0\
    tokenized_lines = []\
    for line in glove_lines:\
        tokenized_line = []\
        for word in line:\
            if word not in word_to_token.keys():\
                embedding_array = np.zeros(300)\
                word_to_token[word] = token_index\
                token_to_word[token_index] = word\
                token_index += 1                \
                try:\
                    embedding_array = embedding_index[word]\
                except KeyError:\
                    pass\
                embedding_matrix.append(embedding_array)\
            tokenized_line.append(word_to_token[word])\
        tokenized_lines.append(tokenized_line)\
    return tokenized_lines, embedding_matrix, token_to_word, word_to_token\
\
def pad_tokenized_lines(tokenized_lines, maxlen):\
    glove_input = []\
    for line in tokenized_lines:\
        line = line + [0] * (maxlen - len(line))\
        assert len(line) == maxlen\
        glove_input.append(np.asarray(line))\
    return np.asarray(glove_input)\
\
#load train data\
train_df = pd.read_csv('gs://bertnglove/train.csv')\
train_df = train_df.sample(frac=0.16,random_state = 42)\
#load test data\
test_df = pd.read_csv('gs://bertnglove/test.csv')\
#text lines\
train_lines = train_df['comment_text'].values\
print(train_lines.shape)\
test_lines = test_df['comment_text'].values\
print(test_lines.shape)\
print('load data done')\
\
train_input, glove_lines = convert_lines(train_lines,maxlen,tokenizer)\
\
glove_lines, embedding_matrix, token_to_word, word_to_token = tokenize_glove_lines(glove_lines, embedding_index)\
\
glove_input = pad_tokenized_lines(glove_lines,maxlen)\
\
AUX_COLUMNS = ['severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']\
TARGET_COLUMN = 'target'\
y_aux_train = train_df[AUX_COLUMNS].values\
num_aux_targets = y_aux_train.shape[-1]\
y_train = train_df[TARGET_COLUMN].values\
\
seg_input = np.zeros((train_input.shape[0],maxlen))\
mask_input = np.ones((train_input.shape[0],maxlen))\
print(seg_input.shape)\
print(mask_input.shape)\
\
embedding_matrix = np.asarray(embedding_matrix)\
model3 = build_marriage_model(embedding_matrix, model, num_aux_targets)\
\
BATCH_SIZE = 512\
model3.fit([train_input, seg_input, mask_input, glove_input],[y_train, y_aux_train],batch_size=BATCH_SIZE, epochs=10,verbose=2)\
\
checkpoint_path = "bertnglove.ckpt"\
model3.save_weights(checkpoint_path)\
\
def tokenize_test_lines(test_lines, word_to_token):\
    tokenized_lines = []\
    for test_line in test_lines:\
        tokenized_line = []\
        for word in test_line:\
            if word in word_to_token.keys():\
                tokenized_line.append(word_to_token[word])\
            else:\
                tokenized_line.append(0)\
        tokenized_lines.append(tokenized_line)\
    return tokenized_lines\
\
test_input, test_glove_lines = convert_lines(test_lines,maxlen,tokenizer)\
test_glove_lines = tokenize_test_lines(test_glove_lines, word_to_token)\
test_glove_input = pad_tokenized_lines(test_glove_lines, maxlen)\
\
hehe = model3.predict([test_input, seg_input, mask_input, test_glove_input],verbose=1,batch_size=BATCH_SIZE)\
submission = pd.read_csv('gs://bertnglove/test.csv', index_col='id')\
submission['prediction'] = hehe[0].flatten()\
submission.reset_index(drop=False, inplace=True)\
submission.to_csv('submission.csv', index=False)}