Search Unity

Question Turnbased Agent with three action

Discussion in 'ML-Agents' started by shah714, Jan 17, 2024.

  1. shah714

    shah714

    Joined:
    Feb 21, 2023
    Posts:
    3
    Hey, I'm creating a simple 2D turn-based fighting game, similar to Final Fantasy. I'm a beginner at this, and I really want to learn and thank everyone for their help. Right now, my game only has three actions for the character to pick from. Unfortunately, I'm struggling to make the character try out different actions, even when the rewards for those actions are better. I wanted to try out with RL so that the trained model will be able to learn about health,magic and about enemy, it should also heal and so on during battle,

    Here is my Agent code:
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using Unity.MLAgents;
    using Unity.MLAgents.Actuators;
    using Unity.MLAgents.Sensors;
    using UnityEngine;

    public class WizardHeroAgent : Agent
    {
    public GameObject meleePrefab;
    public GameObject rangePrefab;
    public GameObject healPrefab;
    private FighterStats enemyFighterStats;

    private GameController gameController; // Reference to GameController

    public override void Initialize()
    {
    Debug.Log("Agent Initialized: " + this.name);

    // Find and store the GameController reference
    gameController = FindObjectOfType<GameController>();
    if (gameController == null)
    {
    Debug.LogError("GameController not found in the scene.");
    }
    }

    public override void OnEpisodeBegin()
    {
    Debug.Log("Episode Started: " + this.name);
    }

    public override void OnActionReceived(ActionBuffers actions)
    {
    // Request a decision only if it's the player's turn
    if (gameController != null && gameController.IsPlayerTurn())
    {


    // Convert discrete actions to integers
    int actionIndex = actions.DiscreteActions[0];

    // Perform the selected action based on the agent's decision
    PerformSelectedAction(actionIndex);

    // The agent's turn will automatically end when the GameController proceeds to the next turn
    }
    }

    // Perform the selected action based on the agent's decision
    private void PerformSelectedAction(int actionIndex)
    {
    switch (actionIndex)
    {
    case 0:
    MeleeAction();
    break;
    case 1:
    RangeAction();
    break;
    case 2:
    HealAction();
    break;
    default:
    Debug.LogError("Invalid action index");
    break;
    }
    }


    public override void Heuristic(in ActionBuffers actionsOut)
    {
    var discreteActionsOut = actionsOut.DiscreteActions;

    // Reset all actions to 0
    discreteActionsOut[0] = 0;
    discreteActionsOut[1] = 0;
    discreteActionsOut[2] = 0;

    // Check for input to set the action during the player's turn
    if (gameController != null && gameController.IsPlayerTurn())
    {
    if (Input.GetKeyDown(KeyCode.Q))
    {
    discreteActionsOut[0] = 1; // Set action 0 to 1 for MeleeAction
    }
    else if (Input.GetKeyDown(KeyCode.W))
    {
    discreteActionsOut[1] = 1; // Set action 1 to 1 for RangeAction
    }
    else if (Input.GetKeyDown(KeyCode.E))
    {
    discreteActionsOut[2] = 1; // Set action 2 to 1 for HealAction
    }
    }
    }


    private void MeleeAction()
    {
    Debug.Log("Performing Melee Action");
    GetComponent<FighterAction>().SelectAttack("melee");
    AddReward(0.1f);
    }

    private void RangeAction()
    {
    Debug.Log("Performing Range Action");
    GetComponent<FighterAction>().SelectAttack("range");
    AddReward(0.2f); // Higher positive reward for using Range
    }

    private void HealAction()
    {
    Debug.Log("Performing Heal Action");
    GetComponent<FighterAction>().SelectAttack("heal");
    AddReward(0.2f); // Higher positive reward for using Heal
    }



    public override void CollectObservations(VectorSensor sensor)
    {
    if (gameController != null && gameController.IsPlayerTurn())
    {
    // Check observing working
    Debug.Log("Observing on Player Turn");

    FighterStats fighterStats = GetComponent<FighterStats>();
    sensor.AddObservation(fighterStats.health);
    sensor.AddObservation(fighterStats.magic);

    AttackScript meleeAttackScript = meleePrefab.GetComponent<AttackScript>();
    AttackScript rangeAttackScript = rangePrefab.GetComponent<AttackScript>();
    AttackScript healAttackScript = healPrefab.GetComponent<AttackScript>();

    if (meleeAttackScript != null)
    {
    sensor.AddObservation(meleeAttackScript.magicCost);
    }

    if (rangeAttackScript != null)
    {
    sensor.AddObservation(rangeAttackScript.magicCost);
    }

    if (healAttackScript != null)
    {
    sensor.AddObservation(healAttackScript.magicCost);
    }
    }
    }
    }
     
  2. shah714

    shah714

    Joined:
    Feb 21, 2023
    Posts:
    3
    Here is my setting for Behavior Parameter
    upload_2024-1-17_14-2-23.png
     

    Attached Files:

  3. smallg2023

    smallg2023

    Joined:
    Sep 2, 2018
    Posts:
    146
    you seem to have your branch settings confused,
    some logic is set to expect 3 branches of size 2
    Code (csharp):
    1. if (Input.GetKeyDown(KeyCode.Q))
    2. {
    3. discreteActionsOut[0] = 1; // Set action 0 to 1 for MeleeAction
    4. }
    5. else if (Input.GetKeyDown(KeyCode.W))
    6. {
    7. discreteActionsOut[1] = 1; // Set action 1 to 1 for RangeAction
    8. }
    9. else if (Input.GetKeyDown(KeyCode.E))
    10. {
    11. discreteActionsOut[2] = 1; // Set action 2 to 1 for HealAction
    12. }
    and other logic is set to look for 1 branch of size 3
    Code (csharp):
    1. public override void OnActionReceived(ActionBuffers actions)
    2. {
    3. // Request a decision only if it's the player's turn
    4. if (gameController != null && gameController.IsPlayerTurn())
    5. {
    6.  
    7.  
    8. // Convert discrete actions to integers
    9. int actionIndex = actions.DiscreteActions[0];
    10.  
    11. // Perform the selected action based on the agent's decision
    12. PerformSelectedAction(actionIndex);
    13.  
    14. // The agent's turn will automatically end when the GameController proceeds to the next turn
    15. }
    16. }
    17.  
    18. // Perform the selected action based on the agent's decision
    19. private void PerformSelectedAction(int actionIndex)
    20. {
    21. switch (actionIndex)
    22. {
    23. case 0:
    24. MeleeAction();
    25. break;
    26. case 1:
    27. RangeAction();
    28. break;
    29. case 2:
    30. HealAction();
    31. break;
    32. default:
    33. Debug.LogError("Invalid action index");
    34. break;
    35. }
    36. }
    stick to 1 method and make sure you can play the game correctly (i would recommend always leaving index 0 of each branch as no action as it's easier to read player input then)
     
    shah714 likes this.
  4. shah714

    shah714

    Joined:
    Feb 21, 2023
    Posts:
    3
    Thanks for the reply and for helping out, the problem is that it should be 1 branch of size 3 and I changed it in behavior parameters and it works