??6??@?J?X?^?}?C?Y???邽?߂́ATensorFlow 2.0?ŐV?̏??????????FTensorFlow 2?{Keras?itf.keras?j????

# ?J?X?^???̊??????֐???Python?֐???????
def custom_activation(x):
    return (tf.exp(x)-tf.exp(-x))/(tf.exp(x)+tf.exp(-x))

# ?J?X?^???̊??????֐??N???X???????i???C???[?̃T?u?N???X???j
# ?itf.keras.layers.Activation()?̑???j
class CustomActivation(tf.keras.layers.Layer):
  def __init__(self, **kwargs):
    super(CustomActivation, self).__init__(**kwargs)

  def call(self, inputs):
    return custom_activation(inputs)

import tensorflow.keras.backend as K

# ?J?X?^???̊??????֐???Python?֐???????
def custom_activation(x):
    return (K.exp(x)-K.exp(-x))/(K.exp(x)+K.exp(-x))

# ?c?c?Ǝ??N???X?̎????͓????Ȃ̂Ŋ????c?c

# ?J?X?^???̑S?????w?i???`?ϊ??j??Python?֐???????
def fully_connected(inputs, weights, bias):
  return tf.matmul(inputs, weights) + bias

# ?J?X?^???̃??C???[?N???X???????i???C???[?̃T?u?N???X???j
# ?itf.keras.layers.Dense()?̑???j
class CustomLayer(tf.keras.layers.Layer):
  def __init__(self, units, input_dim=None, **kwargs):
    self.input_dim = input_dim  # ???͂̎??????i?????C???[?̓??͐??j
    self.units = units          # ?j???[???????i?????C???[?̏o?͐??j
    super(CustomLayer, self).__init__(**kwargs)

  def get_config(self):
    # ???C???[?\?????V???A???C?Y?”\?ɂ??郁?\?b?h?i?Ǝ??̐ݒ荀?ڂ??????o?[?ϐ??Ƃ??Ă???ꍇ?ȂǁA?K?v?ɉ????Ď???????j
    config = super(CustomLayer, self).get_config()
    config.update({
        'input_dim': self.input_dim,
        'units': self.units
    })
    return config

  def build(self, input_shape):
    #print(input_shape) # ???͌`??B?Ⴆ?΁u(2, 2)?v??2?s2??Ȃ???͂̎???????2??
    input_data_dim = input_shape[-1] # ???͂̎??????i?????C???[?̓??͐??j

    # ???͂̎????????`?F?b?N?i?f?o?b?O?j
    if self.input_dim != None:
      assert input_data_dim == self.input_dim  # ?w?肳?ꂽ???͎??????Ǝ??ۂ̓??͎????????قȂ?܂?

    # ?d?݂?lj?????
    self.kernel = self.add_weight(
        shape=(input_data_dim, self.units),
        name='kernel',
        initializer='glorot_uniform',  # ?O?X??̃??X?g1-3?̂悤?ȓƎ??̊֐????w??ł???
        trainable=True)

    # ?o?C?A?X??lj?????
    self.bias = self.add_weight(
        shape=(self.units,),
        name='bias',
        initializer='zeros',
        trainable=True)
    
    #self.built = True # Layer?N???X?Ńr???h?ς݂??ǂ??????Ǘ?????̂Ɏg???Ă???i?Ȃ??Ă??傫?Ȗ??͂Ȃ??j
    super(CustomLayer, self).build(input_shape) # ??Ɠ????Ӗ??BAPI?h?L???????g?Ő???????Ă??鏑????

  def call(self, inputs):
    return fully_connected(inputs, self.kernel, self.bias)

# ?J?X?^???̍œK???A???S???Y???N???X???????i?I?v?e?B?}?C?U?̃T?u?N???X???j
# ?itf.keras.optimizers.SGD?̑???j
class CustomOptimizer(tf.keras.optimizers.Optimizer):
  def __init__(self, learning_rate=0.01, name='CustomOptimizer', **kwargs):
    super(CustomOptimizer, self).__init__(name, **kwargs)
    self.learning_rate = kwargs.get('lr', learning_rate)

  def get_config(self):
    config = super(CustomOptimizer, self).get_config()
    config.update({
        'learning_rate': self.learning_rate
    })
    return config

  def _create_slots(self, var_list):
    for v in var_list: # `Variable`?I?u?W?F?N?g?̃??X?g
      self.add_slot(v, 'accumulator', tf.zeros_like(v))
    # ?Q?l??????B???????ō쐬?????X???b?g?͖??g?p?ɂȂ??Ă???

  def _resource_apply_dense(self, grad, var):
    # ?????ugrad?v?F ???z?i?e???\???j
    # ?????uvar?v?F ?X?V?Ώۂ́u?ϐ??v?????????\?[?X?i?gresource?h?f?[?^?^?̃e???\???j
    # var.device ?̓??e??F /job:localhost/replica:0/task:0/device:CPU:0
    # var.dtype.base_dtype ?̓??e??F <dtype: 'float32'>
    acc = self.get_slot(var, 'accumulator') # ?Q?l??????i?X???b?g?͖??g?p?j
    return var.assign_sub(self.learning_rate * grad)  # ?ϐ??̒l?i?p?????[?^?[?j???X?V
    
  def _resource_apply_sparse(self, grad, var, indices):
    # ?????ugrad?v?F ???z?i?C???f?b?N?X?t???́A?X?p?[?X?ȃe???\???j
    # ?????uvar?v?F ?X?V?Ώۂ́u?ϐ??v?????????\?[?X?i?gresource?h?f?[?^?^?̃e???\???j
    # ?????uindices?v?F ???z???[???ł͂Ȃ??v?f?? ?u?C???f?b?N?X?v?i?????^?̃e???\???j
    raise NotImplementedError("????͎g??Ȃ??̂Ŗ?????")
    # return ?c?c?ϐ??̒l?i?p?????[?^?[?j???X?V????u????v??ԋp????c?c

# ?J?X?^???̑????֐???Python?֐???????
def custom_loss(y_true, y_pred):
    return tf.reduce_mean(tf.square(y_true - y_pred))

# ?J?X?^???̑????֐??N???X???????i???C???[?̃T?u?N???X???j
# ?itf.keras.losses.MeanSquaredError()?̑???j
class CustomLoss(tf.keras.losses.Loss):
  def __init__(self, name="custom_loss", **kwargs):
    super(CustomLoss, self).__init__(name=name, **kwargs)

  def call(self, y_true, y_pred):
    y_pred = tf.convert_to_tensor(y_pred)  # ?O?̂???Tensor??
    y_true = tf.cast(y_true, y_pred.dtype) # ?O?̂??ߓ????f?[?^?^??
    return custom_loss(y_true, y_pred)

# ???????ǂ????𔻒肷??֐???????
def custom_matches(y_true, y_pred):         # y_true?͐????Ay_pred?͗\???i?o?́j
  threshold = tf.cast(0.0, y_pred.dtype)              # -1??1???𕪂???臒l???쐬
  y_pred = tf.cast(y_pred >= threshold, y_pred.dtype) # 臒l??????0?A?ȏ??1?ɕϊ?
  # 2?{????-1.0???邱?ƂŁA0?^1??-1.0?^1.0?ɃX?P?[???ϊ????Đ??𗦂??v?Z
  return tf.equal(y_true, y_pred * 2 - 1.0) # ???????ǂ????̃f?[?^?i???ς͂??Ă??Ȃ??j

# ?J?X?^???̕]???֐??N???X???????i?T?u?N???X???j
# ?itf.keras.metrics.BinaryAccuracy()?̑???j
class CustomAccuracy(tf.keras.metrics.Mean):
  def __init__(self, name='custom_accuracy', dtype=None):
    super(CustomAccuracy, self).__init__(name, dtype)

  # ???𗦂̏?Ԃ??X?V????ۂɌĂяo?????֐????J?X?^?}?C?Y
  def update_state(self, y_true, y_pred, sample_weight=None):
    matches = custom_matches(y_true, y_pred)
    return super(CustomAccuracy, self).update_state(
        matches, sample_weight=sample_weight) # ?????ς͓????Ŏ????I?Ɏ???Ă????

# ???????֐??̒?`
activation1 = CustomActivation(name='activation1')
activation2 = CustomActivation(name='activation2')
acti_out = layers.Activation('tanh', name='acti_out')

# ???f???̐݌v
class NeuralNetwork(tf.keras.Model):
  def __init__(self, *args, **kwargs):
    super(NeuralNetwork, self).__init__(*args, **kwargs)
    self.layer1 = CustomLayer(units=LAYER1_NEURONS, name='layer1')
    self.layer2 = CustomLayer(units=LAYER2_NEURONS, name='layer2')
    self.layer_out = CustomLayer(units=OUTPUT_RESULTS, name='layer_out')
  def call(self, inputs, training=None):
    x1 = activation1(self.layer1(inputs))
    x2 = activation2(self.layer2(x1))
    outputs = acti_out(self.layer_out(x2))
    return outputs

# ???f???̐???
model = NeuralNetwork()

# ???f???̊T?v?ƍ\???}?̏o??
def get_functional_model(model):
  x = layers.Input(shape=(INPUT_FEATURES,), name='layer_in')
  temp_model = tf.keras.Model(inputs=[x], outputs=model.call(x), name='cusotm_model')
  return temp_model
f_model = get_functional_model(model)
f_model.summary()
tf.keras.utils.plot_model(f_model, show_shapes=True, show_layer_names=True, to_file='model.png')
from IPython.display import Image
Image(retina=False, filename='model.png')

# ?œK???A???S???Y???Ƒ????֐??̒?`
optimizer = CustomOptimizer(learning_rate=0.03)
criterion = CustomLoss()

# ?]???֐??i?????Ɛ??𗦁j?̒?`
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = CustomAccuracy(name='train_accuracy')
valid_loss = tf.keras.metrics.Mean(name='valid_loss')
valid_accuracy = CustomAccuracy(name='valid_accuracy')

# ?~?j?o?b?`?p?f?[?^?̏???
X_train_f32 = X_train.astype('float32')
y_train_f32 = y_train.astype('float32')
X_valid_f32 = X_valid.astype('float32')
y_valid_f32 = y_valid.astype('float32')
train_sliced = tf.data.Dataset.from_tensor_slices((X_train_f32, y_train_f32)) # ?P???p
valid_sliced = tf.data.Dataset.from_tensor_slices((X_valid_f32, y_valid_f32)) # ???x???ؗp
train_dataset = train_sliced.shuffle(250).batch(15)
valid_dataset = valid_sliced.batch(15)

import tensorflow.keras.backend as K

# 1?񕪂̌P???i?w?K?j?̏???
@tf.function
def train_step(train_X, train_y):
  training = True
  K.set_learning_phase(training)
  with tf.GradientTape() as tape:
    pred_y = model(train_X, training=training)
    loss = criterion(pred_y, train_y)
  gradient = tape.gradient(loss, model.trainable_weights)
  optimizer.apply_gradients(zip(gradient, model.trainable_weights))
  train_loss(loss)
  train_accuracy(train_y, pred_y)

# 1?񕪂̐??x???؁i?]???j?̏???
@tf.function
def valid_step(valid_X, valid_y):
  training = False
  K.set_learning_phase(training)
  pred_y = model(valid_X, training=training)
  loss = criterion(pred_y, valid_y)
  valid_loss(loss)
  valid_accuracy(valid_y, pred_y)

# ?G?|?b?N?????A?P???i?w?K?j?Ɛ??x???؁i?]???j?????s
EPOCHS = 100
train_history = []
valid_history = []
for epoch in range(EPOCHS):
  train_loss.reset_states()
  train_accuracy.reset_states()
  valid_loss.reset_states()
  valid_accuracy.reset_states()
  for train_X, train_y in train_dataset:
    train_step(train_X, train_y)
  for valid_X, valid_y in valid_dataset:
    valid_step(valid_X, valid_y)   
  n = epoch + 1
  avg_loss = train_loss.result()
  avg_acc = train_accuracy.result()
  avg_val_loss = valid_loss.result()
  avg_val_acc = valid_accuracy.result()
  train_history.append(avg_loss)
  valid_history.append(avg_val_loss)
  print(f'[Epoch {n:3d}/{EPOCHS:3d}]' \
        f' loss: {avg_loss:.5f}, acc: {avg_acc:.5f}' \
        f' val_loss: {avg_val_loss:.5f}, val_acc: {avg_val_acc:.5f}')
print('Finished Training')
print(model.get_weights())

# ?w?K???ʁi?????j?̃O???t??`??
import matplotlib.pyplot as plt
epochs = len(train_history)
plt.plot(range(epochs), train_history, marker='.', label='loss (Training data)')
plt.plot(range(epochs), valid_history, marker='.', label='loss (Validation data)')
plt.legend(loc='best')
plt.grid()
plt.xlabel('epoch')
plt.ylabel('loss')
plt.show()

# ???_
model(tf.constant([[0.1,-0.2]]))
# <tf.Tensor: shape=(1, 1), dtype=float32, numpy=array([[0.99654746]], dtype=float32)> ?c?c?Ȃǂƕ\???????