import numpy as np
 
n_row = 1000
x1 = np.random.randn(n_row)
x2 = np.random.randn(n_row)
x3 = np.random.randn(n_row)
y_classifier = np.array([1 if (x1[i] + x2[i] + (x3[i])/3 + np.random.randn(1) > 1) else 0 for i in range(n_row)])
y_cts = x1 + x2 + x3/3 + np.random.randn(n_row)
dat = np.array([x1, x2, x3]).transpose()

# Take a look at the data
import matplotlib.pyplot as plt
%matplotlib inline
plt.figure(figsize=(12,5))
plt.scatter(dat[:,0],dat[:,1], c=y_classifier)

plt.figure(figsize=(12,5))
plt.scatter(dat[:,0],dat[:,1], c=y_cts)

from sklearn.model_selection import train_test_split

# Split data into test and train
dat_train, dat_test, \
y_classifier_train, y_classifier_test, \
y_cts_train, y_cts_test = train_test_split(dat, y_classifier, y_cts, test_size=.2)

# setup
from keras.models import Input, Model
from keras.layers import Dense

input = Input(shape=(3,), name="input")
output = Dense(1, activation='sigmoid', name='dense')(input)
logistic_model = Model(inputs=input, outputs=output)

# Compile the model 
logistic_model.compile(optimizer='sgd',
                       loss = 'binary_crossentropy',
                       metrics=['accuracy'])

from time import time
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras.callbacks import TensorBoard

# Callbacks monitor the validation loss
# Save the model weights each time the validation loss has improved
callbacks = [
             TensorBoard(log_dir="logs/logistic_model_{}".format(time()), write_grads=False, histogram_freq=32),
             EarlyStopping('val_loss', patience=2), 
             #ModelCheckpoint('logistic_model_weights.h5', save_best_only=True)
            ]

# Fit on training data
logistic_model.optimizer.lr = 0.001
# logistic_model.fit(x=dat_train, y=y_classifier_train, epochs = 1, verbose=2,
#                    validation_data = (dat_test, y_classifier_test))
logistic_history = logistic_model.fit(x=dat_train, y=y_classifier_train, epochs = 500, verbose=0,
                   validation_data = (dat_test, y_classifier_test), callbacks=callbacks)
# logistic_model.fit(x=dat_train, y=y_classifier_train, epochs = 1, verbose=2,
#                    validation_data = (dat_test, y_classifier_test))

# true weights: 1,1,1/3
logistic_model.get_weights()

inputs = Input(shape=(3,))
output = Dense(1, activation='linear')(inputs)
linear_model = Model(inputs, output)
 
linear_model.compile(optimizer='sgd', loss='mse')
callbacks = [
             TensorBoard(log_dir="logs/linear_model_{}".format(time())),
             EarlyStopping('loss', patience=2), 
             #ModelCheckpoint('linear_model_weights.h5', save_best_only=True)
            ]
linear_history = linear_model.fit(x=dat_test, y=y_cts_test, epochs=50, verbose=0, callbacks=callbacks)
# linear_model.fit(x=dat_test, y=y_cts_test, epochs=1, verbose=1)

preds = linear_model.predict(dat_test)
plt.scatter(x=dat_test[:,0], y= np.array(preds) - np.array(y_cts_test).reshape(200,1))

inputs = Input(shape=(3,))
x = Dense(50, activation='relu')(inputs)
output = Dense(1, activation = 'sigmoid')(x)
n_net = Model(inputs, output )

n_net.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

callbacks = [
             TensorBoard(log_dir="logs/n_net_{}".format(time())),
             EarlyStopping('loss', patience=2), 
             #ModelCheckpoint('linear_model_weights.h5', save_best_only=True)
            ]

n_net.fit(x=dat_train, y=y_classifier_train, epochs=50,
verbose=0, validation_data=(dat_test, y_classifier_test), callbacks=callbacks)

from keras.layers import Dropout
# specify how many hidden layers to add (min 1)
n_layers = 5

inputs = Input(shape=(3,))
x = Dense(200, activation='relu')(inputs)
x = Dropout(0.4)(x)
for layer in range(n_layers - 1):
    x = Dense(200, activation='relu')(x)
    x = Dropout(0.3)(x)
output = Dense(1, activation='sigmoid')(x)
deep_n_net = Model(inputs, output)

deep_n_net.compile(optimizer = 'adam', loss= 'binary_crossentropy', metrics=['accuracy'])
callbacks = [
             TensorBoard(log_dir="logs/deep_n_net_{}".format(time())),
             #EarlyStopping('val_loss', patience=2), 
             #ModelCheckpoint('linear_model_weights.h5', save_best_only=True)
            ]
deep_n_net.fit(dat_train, y_classifier_train, epochs = 50, verbose=0,
validation_data = (dat_test, y_classifier_test), callbacks=callbacks)
# deep_n_net.fit(dat_train, y_classifier_train, epochs = 1, verbose=1,
# validation_data = (dat_test, y_classifier_test))

metadata_1 = y_classifier + np.random.gumbel(scale = 0.6, size = n_row)
metadata_2 = y_classifier - np.random.laplace(scale = 0.5, size = n_row)
metadata = np.array([metadata_1,metadata_2]).T
 
# Split data into test and train
dat_train, dat_test, \
y_classifier_train, y_classifier_test, \
y_cts_train, y_cts_test, \
metadata_train, metadata_test = train_test_split(dat, y_classifier, y_cts, metadata, test_size=.2)

from keras.layers import concatenate

input_dat = Input(shape=(3,)) # for the three columns of dat_train
n_net_layer = Dense(50, activation='relu') # first dense layer
x1 = n_net_layer(input_dat)
x1 = Dropout(0.5)(x1)
 
input_metadata = Input(shape=(2,))
x2 = Dense(25, activation= 'relu')(input_metadata)
x2 = Dropout(0.3)(x2)
 
con = concatenate(inputs = [x1,x2] ) # merge in metadata
x3 = Dense(50)(con)
x3 = Dropout(0.3)(x3)
output = Dense(1, activation='sigmoid')(x3)
meta_n_net = Model(inputs=[input_dat, input_metadata], outputs=output)
 
meta_n_net.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['accuracy'])

callbacks = [
             TensorBoard(log_dir="logs/meta_n_net_{}".format(time())),
             #EarlyStopping('val_loss', patience=2), 
             #ModelCheckpoint('linear_model_weights.h5', save_best_only=True)
            ]
meta_n_net.fit(x=[dat_train, metadata_train], y=y_classifier_train, epochs=50, verbose=0,
validation_data=([dat_test, metadata_test], y_classifier_test), callbacks=callbacks)