qlearn.py

#!/usr/bin/env python
from __future__ import print_function

import matplotlib.pyplot as plt
import argparse
import skimage as skimage
from skimage import transform, color, exposure
from skimage.transform import rotate
from skimage.viewer import ImageViewer
import sys
sys.path.append("game/")
import wormy_fun as game
import random
import numpy as np
from collections import deque

import json
from keras import initializations
from keras.initializations import normal, identity
from keras.models import model_from_json
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD , Adam

import pygame
from pygame.locals import *
from time import gmtime, strftime
import os.path
import datetime
import shutil

from keras.utils.visualize_util import plot
counter=1
GAME = 'snake' # the name of the game being played for log files
CONFIG = 'nothreshold'
ACTIONS = 5 # number of valid actions
GAMMA = 0.99 # decay rate of past observations
OBSERVATION = 3200. # timesteps to observe before training
EXPLORE = 3000000. # frames over which to anneal epsilon
FINAL_EPSILON = 0.0001 # final value of epsilon
INITIAL_EPSILON = 0.1 # starting value of epsilon
REPLAY_MEMORY = 50000 # number of previous transitions to remember
BATCH = 32 # size of minibatch
FRAME_PER_ACTION = 1

img_rows , img_cols = 80, 80
#Convert image into Black and white
img_channels = 4 #We stack 4 frames

def buildmodel():
    print("Now we build the model")
    model = Sequential()
    model.add(Convolution2D(32, 8, 8, subsample=(4,4),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same',input_shape=(img_channels,img_rows,img_cols)))
    model.add(Activation('relu'))
    model.add(Convolution2D(64, 4, 4, subsample=(2,2),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same'))
    model.add(Activation('relu'))
    model.add(Convolution2D(64, 3, 3, subsample=(1,1),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same'))
    model.add(Activation('relu'))
    model.add(Flatten())
    model.add(Dense(512, init=lambda shape, name: normal(shape, scale=0.01, name=name)))
    model.add(Activation('relu'))
    model.add(Dense(ACTIONS,init=lambda shape, name: normal(shape, scale=0.01, name=name)))

    adam = Adam(lr=1e-6)
    model.compile(loss='mse',optimizer=adam)
    #model.summary()
    print("We finish building the model")
    plot(model, to_file='model.png', show_shapes=True)
    return model

def trainNetwork(model,args):
    # open up a game state to communicate with emulator

    game_state = game.GameState()
    # store the previous observations in replay memory
    D = deque()

    # get the first state by doing nothing and preprocess the image to 80x80x4
    do_nothing = np.zeros(ACTIONS)
    do_nothing[0] = 1
    x_t, r_0, terminal,_ = game_state.frame_step(do_nothing)

    x_t = skimage.color.rgb2gray(x_t)
    x_t = skimage.transform.resize(x_t,(80,80))
    x_t = skimage.exposure.rescale_intensity(x_t,out_range=(0,255))

    s_t = np.stack((x_t, x_t, x_t, x_t), axis=0)

    #In Keras, need to reshape
    s_t = s_t.reshape(1, s_t.shape[0], s_t.shape[1], s_t.shape[2])

    a_file = open("logs_" + GAME + "/logfile_" + str(counter) + ".txt", 'a')
    learning_mode = 1  # 2 for learng based on human, 3 for reverse reinforcement
    if args['mode'] == 'Run':
        print ("Run mode")
        OBSERVE = 999999999    #We keep observe, never train
        epsilon = FINAL_EPSILON
        print ("Now we load weight")
        model.load_weights("model1.h5")
        adam = Adam(lr=1e-6)
        model.compile(loss='mse',optimizer=adam)
        print ("Weight load successfully")
        training_mode = False  # running
    else:                       #We go to training mode
        OBSERVE = OBSERVATION
        epsilon = INITIAL_EPSILON
        learning_mode=int(args['learning_mode'])

        if os.path.isfile("model1.h5"): #check if file exists.
            model.load_weights("model1.h5")
            adam = Adam(lr=1e-6)
            model.compile(loss='mse',optimizer=adam)
            print ("Weight load successfully")

        os.mkdir("trials" , 0755);
        # printing log file
        training_mode = True # training

        #a_file.write("init param:" + "\n")
        #a_file.write("start time:" + strftime("%Y-%m-%d %H:%M:%S", gmtime()) + "\n")
        #a_file.write("Game:" + GAME + "\n" + "GAMMA:" + str(GAMMA) + "\n" + "OBSERVATION:" + str(OBSERVATION) + "\n")
        #a_file.write("Initial epsilon:" + str(INITIAL_EPSILON) + "\n" +"Final epsilon:" + str(FINAL_EPSILON) + "\n")
        #a_file.write("Explore:" + str(EXPLORE) + "\n" + "Replay memory:" + str(REPLAY_MEMORY) + "\n" + "\n")


    j = 0
    t = 0
    start_time=datetime.datetime.now()
    num_folder=0
    high_score=0

    while (True):
        loss = 0
        Q_sa = 0
        action_index = 0
        r_t = 0
        a_t = np.zeros([ACTIONS])

        #choose an action epsilon greedy
        if t % FRAME_PER_ACTION == 0:
            if not training_mode: #running
                q = model.predict(s_t)  # input a stack of 4 images, get the prediction
                max_Q = np.argmax(q)
                action_index = max_Q
                a_t[max_Q] = 1

            elif random.random() <= epsilon:
                #print("----------Random Action----------")
                #raw_input("Press Enter to continue...")
                action_index = random.randrange(ACTIONS)
                a_t[action_index] = 1

            elif (learning_mode == 1):
                q = model.predict(s_t)  # input a stack of 4 images, get the prediction
                max_Q = np.argmax(q)
                action_index = max_Q
                a_t[max_Q] = 1

            else:  # learning_mode is 2 or 3
                # action_index = 0 do nothing
                # action_index = 1 up

                if (learning_mode == 3):
                    # comp chooses his next move
                    q = model.predict(s_t)  # input a stack of 4 images, get the prediction
                    max_Q = np.argmax(q)
                    temp_action_index = max_Q

                for event in pygame.event.get():  # event handling loop

                    if event.type == KEYDOWN:
                        if event.key in (K_UP, K_w):
                            action_index = 1

                        elif event.key == K_ESCAPE:
                            learning_mode = 1
                            print("end learning via human:", strftime("%Y-%m-%d %H:%M:%S", gmtime()), file=a_file)

                    else:
                        action_index = 0

                a_t[action_index] = 1

        #We reduced the epsilon gradually
        if epsilon > FINAL_EPSILON and t > OBSERVE:
            epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE

        #run the selected action and observed next state and reward
        x_t1_colored, r_t, terminal,score = game_state.frame_step(a_t)

        game_over=terminal
        if(learning_mode==3):
            if (action_index == temp_action_index):
                r_t=1
            else:
                r_t=-1
                print ("my action is :",action_index, "comp action is:", temp_action_index)
                action_index==temp_action_index

        x_t1 = skimage.color.rgb2gray(x_t1_colored)
        x_t1 = skimage.transform.resize(x_t1,(80,80))
        x_t1 = skimage.exposure.rescale_intensity(x_t1, out_range=(0, 255))

        x_t1 = x_t1.reshape(1, 1, x_t1.shape[0], x_t1.shape[1])
        s_t1 = np.append(x_t1, s_t[:, :3, :, :], axis=1)

        # store the transition in D
        D.append((s_t, action_index, r_t, s_t1, terminal))
        if len(D) > REPLAY_MEMORY:
            D.popleft()

        #only train if done observing
        if t > OBSERVE:
            #sample a minibatch to train on
            minibatch = random.sample(D, BATCH)

            inputs = np.zeros((BATCH, s_t.shape[1], s_t.shape[2], s_t.shape[3]))   #32, 80, 80, 4
            targets = np.zeros((inputs.shape[0], ACTIONS))                         #32, 2

            #Now we do the experience replay
            for i in range(0, len(minibatch)):
                state_t = minibatch[i][0]
                action_t = minibatch[i][1]   #This is action index
                reward_t = minibatch[i][2]
                state_t1 = minibatch[i][3]
                terminal = minibatch[i][4]
                # if terminated, only equals reward

                inputs[i:i + 1] = state_t    #I saved down s_t

                targets[i] = model.predict(state_t)  # Hitting each buttom probability
                Q_sa = model.predict(state_t1)

                if terminal or learning_mode==3:
                    targets[i, action_t] = reward_t
                else:
                    targets[i, action_t] = reward_t + GAMMA * np.max(Q_sa)

            # targets2 = normalize(targets)
            loss += model.train_on_batch(inputs, targets)
        s_t = s_t1
        t = t + 1

        # save progress every 10000 iterations
        if t % 100 == 0:
            #print("Now we save model")
            model.save_weights("model1.h5", overwrite=True)
            with open("model.json", "w") as outfile:
                json.dump(model.to_json(), outfile)

        # print info
        state = ""
        if t <= OBSERVE:
            state = "observe"
        elif t > OBSERVE and t <= OBSERVE + EXPLORE:
            state = "explore"
        else:
            state = "train"

        print("TIMESTEP", t, "/ STATE", state, \
            "/ EPSILON", epsilon, "/ ACTION", action_index, "/ REWARD", r_t, \
            "/ Q_MAX " , np.max(Q_sa), "/ Loss ", loss)

	current_time=datetime.datetime.now()
        elapsedTime = (current_time - start_time).total_seconds()  


	if(elapsedTime>=30*60):	
		num_folder+=1
		start_time=datetime.datetime.now()
		os.mkdir("trials/" + str (num_folder), 0755);
		shutil.copy2('model1.h5', 'trials/' + str (num_folder) + '/model1.h5')

        if (game_over) and (learning_mode == 1):
            j = j + 1
            
            if(training_mode) and score>high_score :
                a_file.write(str(j) +  " score: " + str(score) + " time:" +str(current_time) + "\n")
                a_file.flush()
		high_score=score





    print("Episode finished!")
    print("************************")

    finish_time = time.clock()
    print("finish time:", strftime("%Y-%m-%d %H:%M:%S", gmtime()), file=a_file)
    a_file.close

def playGame(args):
    model = buildmodel()
    trainNetwork(model,args)

def main():
    parser = argparse.ArgumentParser(description='Description of your program')
    parser.add_argument('-m','--mode', help='Train / Run', required=True)
    parser.add_argument('-l','--learning_mode', help='1,2,3,4', required=False)
    args = vars(parser.parse_args())
    playGame(args)

if __name__ == "__main__":
    main()