I am trying to use bert pretrained model for intent classification. here is my code in jupyter notebok.
class DataPreparation: text_column = "text" label_column = "intent" def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192): self.tokenizer = tokenizer self.max_seq_len = 0 self.classes = classes ((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self.prepare_data, [train, test]) print("max seq_len", self.max_seq_len) self.max_seq_len = min(self.max_seq_len, max_seq_len) self.train_x, self.test_x = map(self.data_padding, [self.train_x, self.test_x]) def prepare_data(self, df): x, y = [], [] for _, row in tqdm(df.iterrows()): text, label = row[DataPreparation.text_column], row[DataPreparation.label_column] tokens = self.tokenizer.tokenize(text) tokens = ["[CLS]"] + tokens + ["[SEP]"] token_ids = self.tokenizer.convert_tokens_to_ids(tokens) self.max_seq_len = max(self.max_seq_len, len(token_ids)) x.append(token_ids) y.append(self.classes.index(label)) return np.array(x), np.array(y) def data_padding(self, ids): x = [] for input_ids in ids: input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)] input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids)) x.append(np.array(input_ids)) return np.array(x)tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))def model_defination(max_seq_len, bert_ckpt_file): with tf.io.gfile.GFile(bert_config_file, "r") as reader: bc = StockBertConfig.from_json_string(reader.read()) bert_params = map_stock_config_to_params(bc) bert_params.adapter_size = None bert = BertModelLayer.from_params(bert_params, name="bert") input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids") bert_output = bert(input_ids) print("bert shape", bert_output.shape) cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output) cls_out = keras.layers.Dropout(0.5)(cls_out) logits = keras.layers.Dense(units=768, activation="tanh")(cls_out) logits = keras.layers.Dropout(0.5)(logits) logits = keras.layers.Dense(units=len(classes), activation="softmax")(logits) model = keras.Model(inputs=input_ids, outputs=logits) model.build(input_shape=(None, max_seq_len)) load_stock_weights(bert, bert_ckpt_file) return modelclasses = train.intent.unique().tolist()data = DataPreparation(train, test, tokenizer, classes, max_seq_len=128)data.train_x.shapedata.train_y[0]model = model_defination(data.max_seq_len, bert_ckpt_file)Now when I am trying to call the function, I am getting error. The parameter values have max_seq_len = 55, bert_ckpt_file = bert checkpoint file.
when i create the model i am getting the below error:
TypeError Traceback (most recent call last)<ipython-input-17-af3e534b3882> in <module>----> 1 model = model_defination(data.max_seq_len, bert_ckpt_file)<ipython-input-16-a83a622dafe3> in model_defination(max_seq_len, bert_ckpt_file) 9 input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids") 10 #input_spec = tf.keras.layers.InputSpec(ndim=3)---> 11 bert_output = bert(input_ids) 12 13 print("bert shape", bert_output.shape)~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in __call__(self, *args, **kwargs) 974 # >> model = tf.keras.Model(inputs, outputs) 975 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):--> 976 return self._functional_construction_call(inputs, args, kwargs, 977 input_list) 978 ~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list) 1112 layer=self, inputs=inputs, build_graph=True, training=training_value): 1113 # Check input assumptions set after layer building, e.g. input shape.-> 1114 outputs = self._keras_tensor_symbolic_call( 1115 inputs, input_masks, args, kwargs) 1116 ~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs) 846 return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature) 847 else:--> 848 return self._infer_output_signature(inputs, args, kwargs, input_masks) 849 850 def _infer_output_signature(self, inputs, args, kwargs, input_masks):~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks) 886 self._maybe_build(inputs) 887 inputs = self._maybe_cast_inputs(inputs)--> 888 outputs = call_fn(inputs, *args, **kwargs) 889 890 self._handle_activity_regularization(inputs, outputs)~\Anaconda3\lib\site-packages\tensorflow\python\autograph\impl\api.py in wrapper(*args, **kwargs) 693 except Exception as e: # pylint:disable=broad-except 694 if hasattr(e, 'ag_error_metadata'):--> 695 raise e.ag_error_metadata.to_exception(e) 696 else: 697 raiseTypeError: in user code: C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\model.py:80 call * output = self.encoders_layer(embedding_output, mask=mask, training=training) C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1030 __call__ ** self._maybe_build(inputs) C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2659 _maybe_build self.build(input_shapes) # pylint:disable=not-callable C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\transformer.py:209 build self.input_spec = keras.layers.InputSpec(shape=input_shape) C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2777 __setattr__ super(tf.__internal__.tracking.AutoTrackable, self).__setattr__(name, value) # pylint: disable=bad-super-call C:\Users\kamrul.moin\Anaconda3\lib\site-packages\tensorflow\python\training\tracking\base.py:530 _method_wrapper result = method(self, *args, **kwargs) C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1296 input_spec raise TypeError('Layer input_spec must be an instance of InputSpec. ' TypeError: Layer input_spec must be an instance of InputSpec. Got: InputSpec(shape=(None, 55, 768), ndim=3)