Raycast not always working

I have this code, it's a bit of a mess I know but that's the best I could do.
Anyway, raycast works just when I'm touching the object, this is it, I see nothing wrong in it :|

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. public class gunHandling : MonoBehaviour {
5.  
6.     public AudioClip gunshot;
7.     public GameObject rayStartPoint;
8.     //original coords 0.0281446 -0.02998793 0.03127646
9.     //new coords -0.09138346 -0.005160332 0.03127646
10.     // Use this for initialization
11.     public Renderer muzzleFlash;
12.     public Light muzzleLight;
13.     public Material[] muzzleFlashes;
14.     float fireRate = 1 / 14;
15.     Quaternion originalRot;
16.     int bulletCount = 31;
17.     bool isShooting = false;
18.     float moveAmount = 5.0f;
19.     float moveX, moveY;
20.     bool isReloading = false;
21.     Vector3 originalPos, newPos;
22.     public Camera playerCamera;
23.     Vector3 mouseGunPos;
24.     Quaternion originalCameraPos, recoilPos;
25.     float recoilAmount = 10;
26.     float originalFOV;
27.     public Animator anim;
28.     double X = -0.09138346, Y = -0.005160332, Z = 0.03127646;
29.  
30.     void Start () {
31.         muzzleLight.color = Color.yellow;
32.         muzzleFlash.enabled = false;
33.         muzzleLight.enabled = false;
34.         originalCameraPos = Camera.main.transform.rotation;
35.         originalFOV = Camera.main.fieldOfView;
36.         originalPos = transform.localPosition;
37.         newPos.x = (float)X;
38.         newPos.y = (float)Y;
39.         newPos.z = (float)Z;
40.         originalRot = transform.localRotation;
41.     }
42.    
43.     // Update is called once per frame
44.  
45.     void Update () {
46.         var forward = transform.TransformDirection(Vector3.forward);
47.         float addToY = transform.localPosition.y;
48.         float addToX = transform.localPosition.x;
49.         Ray rayCast = new Ray(rayStartPoint.transform.position, rayStartPoint.transform.forward);
50.        
51.         RaycastHit hit;
52.         Debug.DrawRay(rayCast.origin, rayCast.direction * 100, Color.yellow);
53.         if(Input.GetKey(KeyCode.Mouse1) && !isReloading)
54.         {
55.             playerCamera.fieldOfView = Mathf.Lerp(playerCamera.fieldOfView, 25.0f, 6f * Time.deltaTime);
56.             transform.localPosition = Vector3.Lerp(transform.localPosition, newPos, 6f * Time.deltaTime);
57.         }
58.         else //if(canChangeFOV)
59.         {
60.             moveX = Input.GetAxis("Mouse X") * Time.deltaTime * moveAmount;
61.             moveY = Input.GetAxis("Mouse Y") * Time.deltaTime * moveAmount;
62.  
63.             mouseGunPos = new Vector3(originalPos.x + moveX, originalPos.y + moveY, originalPos.z);
64.             transform.localPosition = Vector3.Lerp(transform.localPosition, mouseGunPos, 4 * Time.deltaTime);
65.             playerCamera.fieldOfView = Mathf.Lerp(playerCamera.fieldOfView, originalFOV, 6f * Time.deltaTime);
66.             transform.localPosition = Vector3.Lerp(transform.localPosition, originalPos, 6f * Time.deltaTime);
67.         }
68.         if (Input.GetKey(KeyCode.Mouse0) && !isReloading && !isShooting)
69.         {
70.             muzzleFlash.renderer.material = muzzleFlashes[Random.Range(0, 2)];
71.             StartCoroutine(fire());
72.             addToY += Random.Range(-0.1f, 0.1f);
73.             addToX += Random.Range(-0.1f, 0.1f);
74.             transform.localPosition = new Vector3(transform.localPosition.x, addToY * Time.deltaTime, transform.localPosition.z - Random.Range(-0.5f, 0.1f) * Time.deltaTime);
75.             if(Physics.Raycast(rayCast, out hit, 1000))
76.             {
77.                 if(hit.collider.gameObject.tag == "Enemy")
78.                 {
79.                     Debug.Log("Hit!");
80.                     Destroy(hit.collider.gameObject);
81.                 }
82.             }
83.             addToY = 0;
84.         }
85.         if(bulletCount <= 0 || Input.GetKey(KeyCode.R))
86.         {
87.             isReloading = true;
88.             StartCoroutine(reload());
89.         }
90.     }
91.     IEnumerator fire()
92.     {
93.         audio.PlayOneShot(gunshot);
94.         muzzleFlash.renderer.enabled = true;
95.         muzzleLight.enabled = true;
96.         isShooting = true;
97.         bulletCount--;
98.         yield return new WaitForSeconds(fireRate + 0.06f);
99.         isShooting = false;
100.         muzzleFlash.renderer.enabled = false;
101.         muzzleLight.enabled = false;
102.     }
103.    
104.     IEnumerator reload()
105.     {
106.         anim.SetBool("IsReloading", true);
107.         //anim.SetBool("IsReloading", false);
108.         bulletCount = 30;
109.         yield return new WaitForSeconds(3.0f);
110.         isReloading = false;
111.         anim.SetBool("IsReloading", false);
112.         transform.localRotation = originalRot;
113.     }
114. }
115.  

 

I found the reason, I have a cube with a material on it which is supposed to be the muzzle flash, unfortunately the cube is blocking the raycast...any ways to fix that?

EDIT: Fixed by removing its box collider.

 

Or you could just set it's layer to "Ignore Raycast"

 

Oh, well. Disabling it is better IMO as it slowed down the movement when looking at the ground.