问题描述
我在 Java 中尝试使用 Q-Learning(强化学习)进行 Pacman 游戏。 但是,我可以看到游戏自动暂停了几秒钟,然后再次运行。我只是想知道这是什么原因。
Youtube 视频链接 https://www.youtube.com/watch?v=bmDTAX13so4&feature=youtu.be
代码 https://github.com/iamarkaj/pacman-Q_learning 的 Github 链接
Board.java :
package com.zetcode;
import java.util.Random;
import java.awt.AWTException;
import java.awt.Basicstroke;
import java.awt.Color;
import java.awt.Dimension;
import java.awt.Font;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.KeyAdapter;
import java.awt.event.KeyEvent;
import javax.swing.ImageIcon;
import javax.swing.JPanel;
import javax.swing.Timer;
public class Board extends JPanel implements ActionListener {
/**
*
*/
private static final long serialVersionUID = 1L;
private Dimension d;
private final Font smallFont = new Font("Helvetica",Font.BOLD,14);
private Image ii;
private final Color dotColor = new Color(192,192,0);
private Color mazeColor;
private boolean inGame = true;
private boolean dying = false;
private final int BLOCK_SIZE = 24;
private final int N_BLOCKS = 15;
private final int SCREEN_SIZE = N_BLOCKS * BLOCK_SIZE;
private final int PAC_ANIM_DELAY = 2;
private final int PACMAN_ANIM_COUNT = 4;
private final int MAX_GHOSTS = 12;
private final int PACMAN_SPEED = 6;
private int pacAnimCount = PAC_ANIM_DELAY;
private int pacAnimDir = 1;
private int pacmanAnimPos = 0;
private int N_GHOSTS = 6;
private int score;
private int[] dx,dy;
private int[] ghost_x,ghost_y,ghost_dx,ghost_dy,ghostSpeed;
private Image ghost;
private Image pacman1,pacman2up,pacman2left,pacman2right,pacman2down;
private Image pacman3up,pacman3down,pacman3left,pacman3right;
private Image pacman4up,pacman4down,pacman4left,pacman4right;
private int pacman_x,pacman_y,pacmand_x,pacmand_y;
private int req_dx,req_dy,view_dx,view_dy;
private final short levelData[] = {
19,26,18,22,21,17,16,20,24,25,28,1,19,9,8,28
};
private final int validSpeeds[] = {1,2,3,4,6,8};
private final int maxSpeed = 6;
private int currentSpeed = 3;
private short[] screenData;
private Timer timer;
private int[][][] Q = new int[60][60][6];
private int action;
private double lr;
private int step = 0;
private int x_state;
private int y_state;
private int key = 1;
private int seed = 0;
public Board() {
loadImages();
initvariables();
initBoard();
}
private void initBoard() {
addKeyListener(new TAdapter());
setFocusable(true);
setBackground(Color.black);
}
private void initvariables() {
screenData = new short[N_BLOCKS * N_BLOCKS];
mazeColor = new Color(5,100,5);
d = new Dimension(400,400);
ghost_x = new int[MAX_GHOSTS];
ghost_dx = new int[MAX_GHOSTS];
ghost_y = new int[MAX_GHOSTS];
ghost_dy = new int[MAX_GHOSTS];
ghostSpeed = new int[MAX_GHOSTS];
dx = new int[4];
dy = new int[4];
timer = new Timer(40,this);
timer.start();
}
@Override
public void addNotify() {
super.addNotify();
initGame();
}
private void doAnim() {
pacAnimCount--;
if (pacAnimCount <= 0) {
pacAnimCount = PAC_ANIM_DELAY;
pacmanAnimPos = pacmanAnimPos + pacAnimDir;
if (pacmanAnimPos == (PACMAN_ANIM_COUNT - 1) || pacmanAnimPos == 0) {
pacAnimDir = -pacAnimDir;
}
}
}
private void playGame(Graphics2D g2d) {
if (dying) {
death();
} else {
movePacman();
drawPacman(g2d);
moveGhosts(g2d);
checkMaze();
}
}
private void drawscore(Graphics2D g) {
String s;
g.setFont(smallFont);
g.setColor(new Color(96,128,255));
s = "score: " + score;
g.drawString(s,SCREEN_SIZE / 2 + 96,SCREEN_SIZE + 16);
}
private void checkMaze() {
short i = 0;
boolean finished = true;
while (i < N_BLOCKS * N_BLOCKS && finished) {
if ((screenData[i] & 48) != 0) {
finished = false;
}
i++;
}
if (finished) {
score += 50;
if (N_GHOSTS < MAX_GHOSTS) {
N_GHOSTS++;
}
if (currentSpeed < maxSpeed) {
currentSpeed++;
}
initLevel();
}
}
private void death() {
inGame = true;
initLevel();
}
private void moveGhosts(Graphics2D g2d) {
short i;
int pos;
int count;
for (i = 0; i < N_GHOSTS; i++) {
if (ghost_x[i] % BLOCK_SIZE == 0 && ghost_y[i] % BLOCK_SIZE == 0) {
pos = ghost_x[i] / BLOCK_SIZE + N_BLOCKS * (int) (ghost_y[i] / BLOCK_SIZE);
count = 0;
if ((screenData[pos] & 1) == 0 && ghost_dx[i] != 1) {
dx[count] = -1;
dy[count] = 0;
count++;
}
if ((screenData[pos] & 2) == 0 && ghost_dy[i] != 1) {
dx[count] = 0;
dy[count] = -1;
count++;
}
if ((screenData[pos] & 4) == 0 && ghost_dx[i] != -1) {
dx[count] = 1;
dy[count] = 0;
count++;
}
if ((screenData[pos] & 8) == 0 && ghost_dy[i] != -1) {
dx[count] = 0;
dy[count] = 1;
count++;
}
if (count == 0) {
if ((screenData[pos] & 15) == 15) {
ghost_dx[i] = 0;
ghost_dy[i] = 0;
} else {
ghost_dx[i] = -ghost_dx[i];
ghost_dy[i] = -ghost_dy[i];
}
} else {
Random rand = new Random(seed);
seed += 1;
double prob2 = rand.nextInt(10)+2;
double prob2_ = 1/prob2;
count = (int) (prob2_ * count);
if (count > 3) {
count = 3;
}
ghost_dx[i] = dx[count];
ghost_dy[i] = dy[count];
}
}
ghost_x[i] = ghost_x[i] + (ghost_dx[i] * ghostSpeed[i]);
ghost_y[i] = ghost_y[i] + (ghost_dy[i] * ghostSpeed[i]);
drawGhost(g2d,ghost_x[i] + 1,ghost_y[i] + 1);
if (pacman_x > (ghost_x[i] - 12) && pacman_x < (ghost_x[i] + 12)
&& pacman_y > (ghost_y[i] - 12) && pacman_y < (ghost_y[i] + 12)
&& inGame) {
dying = true;
}
}
}
private void drawGhost(Graphics2D g2d,int x,int y) {
g2d.drawImage(ghost,x,y,this);
}
private void movePacman() {
int pos;
short ch;
if (req_dx == -pacmand_x && req_dy == -pacmand_y) {
pacmand_x = req_dx;
pacmand_y = req_dy;
view_dx = pacmand_x;
view_dy = pacmand_y;
}
if (pacman_x % BLOCK_SIZE == 0 && pacman_y % BLOCK_SIZE == 0) {
pos = pacman_x / BLOCK_SIZE + N_BLOCKS * (int) (pacman_y / BLOCK_SIZE);
ch = screenData[pos];
if ((ch & 16) != 0) {
screenData[pos] = (short) (ch & 15);
score++;
}
if (req_dx != 0 || req_dy != 0) {
if (!((req_dx == -1 && req_dy == 0 && (ch & 1) != 0)
|| (req_dx == 1 && req_dy == 0 && (ch & 4) != 0)
|| (req_dx == 0 && req_dy == -1 && (ch & 2) != 0)
|| (req_dx == 0 && req_dy == 1 && (ch & 8) != 0))) {
pacmand_x = req_dx;
pacmand_y = req_dy;
view_dx = pacmand_x;
view_dy = pacmand_y;
}
}
// Check for standstill
if ((pacmand_x == -1 && pacmand_y == 0 && (ch & 1) != 0)
|| (pacmand_x == 1 && pacmand_y == 0 && (ch & 4) != 0)
|| (pacmand_x == 0 && pacmand_y == -1 && (ch & 2) != 0)
|| (pacmand_x == 0 && pacmand_y == 1 && (ch & 8) != 0)) {
pacmand_x = 0;
pacmand_y = 0;
}
}
pacman_x = pacman_x + PACMAN_SPEED * pacmand_x;
pacman_y = pacman_y + PACMAN_SPEED * pacmand_y;
x_state = pacman_x*60/380;
y_state = pacman_y*60/420;
step += 1;
if (step % 500 == 0) {
System.out.println(step);
}
lr = Math.pow(0.85,(step/100));
Q[x_state][y_state][action] += lr * (score + Q[x_state][y_state][action]);
}
private void drawPacman(Graphics2D g2d) {
if (view_dx == -1) {
drawPacmanLeft(g2d);
} else if (view_dx == 1) {
drawPacmanRight(g2d);
} else if (view_dy == -1) {
drawPacmanUp(g2d);
} else {
drawPacmanDown(g2d);
}
}
private void drawPacmanUp(Graphics2D g2d) {
switch (pacmanAnimPos) {
case 1:
g2d.drawImage(pacman2up,pacman_x + 1,pacman_y + 1,this);
break;
case 2:
g2d.drawImage(pacman3up,this);
break;
case 3:
g2d.drawImage(pacman4up,this);
break;
default:
g2d.drawImage(pacman1,this);
break;
}
}
private void drawPacmanDown(Graphics2D g2d) {
switch (pacmanAnimPos) {
case 1:
g2d.drawImage(pacman2down,this);
break;
case 2:
g2d.drawImage(pacman3down,this);
break;
case 3:
g2d.drawImage(pacman4down,this);
break;
}
}
private void drawPacmanLeft(Graphics2D g2d) {
switch (pacmanAnimPos) {
case 1:
g2d.drawImage(pacman2left,this);
break;
case 2:
g2d.drawImage(pacman3left,this);
break;
case 3:
g2d.drawImage(pacman4left,this);
break;
}
}
private void drawPacmanRight(Graphics2D g2d) {
switch (pacmanAnimPos) {
case 1:
g2d.drawImage(pacman2right,this);
break;
case 2:
g2d.drawImage(pacman3right,this);
break;
case 3:
g2d.drawImage(pacman4right,this);
break;
}
}
private void drawMaze(Graphics2D g2d) {
short i = 0;
int x,y;
for (y = 0; y < SCREEN_SIZE; y += BLOCK_SIZE) {
for (x = 0; x < SCREEN_SIZE; x += BLOCK_SIZE) {
g2d.setColor(mazeColor);
g2d.setstroke(new Basicstroke(2));
if ((screenData[i] & 1) != 0) {
g2d.drawLine(x,y + BLOCK_SIZE - 1);
}
if ((screenData[i] & 2) != 0) {
g2d.drawLine(x,x + BLOCK_SIZE - 1,y);
}
if ((screenData[i] & 4) != 0) {
g2d.drawLine(x + BLOCK_SIZE - 1,y + BLOCK_SIZE - 1);
}
if ((screenData[i] & 8) != 0) {
g2d.drawLine(x,y + BLOCK_SIZE - 1,y + BLOCK_SIZE - 1);
}
if ((screenData[i] & 16) != 0) {
g2d.setColor(dotColor);
g2d.fillRect(x + 11,y + 11,2);
}
i++;
}
}
}
private void initGame() {
score = 0;
initLevel();
N_GHOSTS = 6;
currentSpeed = 3;
}
private void initLevel() {
int i;
for (i = 0; i < N_BLOCKS * N_BLOCKS; i++) {
screenData[i] = levelData[i];
}
continueLevel();
}
private void continueLevel() {
score = 0;
short i;
int dx = 1;
int random;
for (i = 0; i < N_GHOSTS; i++) {
ghost_y[i] = 4 * BLOCK_SIZE;
ghost_x[i] = 4 * BLOCK_SIZE;
ghost_dy[i] = 0;
ghost_dx[i] = dx;
dx = -dx;
Random rand = new Random(0);
double prob1 = rand.nextInt(10)+2;
double prob1_ = 1/prob1;
random = (int) (prob1_ * (currentSpeed + 1));
if (random > currentSpeed) {
random = currentSpeed;
}
ghostSpeed[i] = validSpeeds[random];
}
pacman_x = 7 * BLOCK_SIZE;
pacman_y = 11 * BLOCK_SIZE;
pacmand_x = 0;
pacmand_y = 0;
req_dx = 0;
req_dy = 0;
view_dx = -1;
view_dy = 0;
dying = false;
}
private void loadImages() {
ghost = new ImageIcon("src/resources/images/ghost.png").getimage();
pacman1 = new ImageIcon("src/resources/images/pacman.png").getimage();
pacman2up = new ImageIcon("src/resources/images/up1.png").getimage();
pacman3up = new ImageIcon("src/resources/images/up2.png").getimage();
pacman4up = new ImageIcon("src/resources/images/up3.png").getimage();
pacman2down = new ImageIcon("src/resources/images/down1.png").getimage();
pacman3down = new ImageIcon("src/resources/images/down2.png").getimage();
pacman4down = new ImageIcon("src/resources/images/down3.png").getimage();
pacman2left = new ImageIcon("src/resources/images/left1.png").getimage();
pacman3left = new ImageIcon("src/resources/images/left2.png").getimage();
pacman4left = new ImageIcon("src/resources/images/left3.png").getimage();
pacman2right = new ImageIcon("src/resources/images/right1.png").getimage();
pacman3right = new ImageIcon("src/resources/images/right2.png").getimage();
pacman4right = new ImageIcon("src/resources/images/right3.png").getimage();
}
@Override
public void paintComponent(Graphics g) {
super.paintComponent(g);
doDrawing(g);
}
private void doDrawing(Graphics g) {
Graphics2D g2d = (Graphics2D) g;
g2d.setColor(Color.black);
g2d.fillRect(0,d.width,d.height);
drawMaze(g2d);
drawscore(g2d);
doAnim();
if (inGame) {
playGame(g2d);
}
g2d.drawImage(ii,5,this);
Toolkit.getDefaultToolkit().sync();
g2d.dispose();
}
class TAdapter extends KeyAdapter {
public void keypressed(KeyEvent e) {
Robot r = null;
try {
r = new Robot();
} catch (AWTException e1) {
// Todo Auto-generated catch block
e1.printstacktrace();
}
r.keyPress(KeyEvent.VK_ENTER);
r.keyrelease(KeyEvent.VK_ENTER);
Random rand = new Random();
int prob = rand.nextInt(100)+1;
if (step < 500000) {
if (prob < 90) {
key = rand.nextInt(5)+1;
action = key;}
else {
int max = Q[x_state][y_state][0];
action = 1;
for (int i = 0; i < 6; i++) {
if(Q[x_state][y_state][i] > max) {
action = i;
key = i;}
}
}
}
else {
int max = Q[x_state][y_state][0];
action = 1;
for (int i = 0; i < 6; i++) {
if(Q[x_state][y_state][i] > max) {
action = i;}
}
}
if (inGame) {
if (key == 1) {
req_dx = -1;
req_dy = 0;
} else if (key == 2) {
req_dx = 1;
req_dy = 0;
} else if (key == 3) {
req_dx = 0;
req_dy = -1;
} else if (key == 4) {
req_dx = 0;
req_dy = 1;
}
}
}
}
@Override
public void actionPerformed(ActionEvent e) {
repaint();
}
}
吃豆子.java:
import java.awt.EventQueue;
import javax.swing.JFrame;
public class Pacman extends JFrame {
/**
*
*/
private static final long serialVersionUID = 1L;
public Pacman() {
initUI();
}
private void initUI() {
add(new Board());
setTitle("Pacman");
setDefaultCloSEOperation(EXIT_ON_CLOSE);
setSize(380,420);
setLocationRelativeto(null);
}
public static void main(String[] args) {
EventQueue.invokelater(() -> {
var ex = new Pacman();
ex.setVisible(true);
});
}
}
提前致谢。
解决方法
看起来像是优化问题,我确实发现了一些问题:
-
例如,您不需要每次都绘制所有内容
g.setColor(Color.black);
g.fillRect(0,d.width,d.height);
只能调用一次。或者什么时候重新创建迷宫。
-
drawMaze - 真的有可能每个 if 语句都被触发吗?如果不是,则使用 if else 顺序,这将更频繁地发生,而不是更频繁地发生
-
您可能应该放弃使用 Graphics2D,因为它的性能低于开箱即用的 Graphics。或者在使用时阅读有关性能的更多信息
-
对于我来说删除 Toolkit.getDefaultToolkit().sync();提高性能(减少暂停)
所以总的来说,这都是关于性能的,尝试将graphics2d更改为graphics并用正方形替换pacman和ghost,您会看到它是否更好
编辑:我之前玩过一些蛇 AI,不确定它是否对你有帮助,但你可以在这里查看:https://github.com/Morph21/AI
这不是最漂亮的代码,所以不要评判我