Bug Position of a game object sometimes not updating

So in my game I gave the player the ability to grab and manipulate objects in the scene with physics. The issue is that, when I play the game and attempt to grab an object, the "pivot", which is a game object that has it's position set to the raycast hit point every time the "Grab()" function is called, the position sometimes doesn't update. And no, it has nothing to do with local/global scale interference because I've done intensive debugging and it had nothing to do with it. Also, the pivot becomes the parent of the object I want to grab so I can achieve the illusion that the object is getting manipulated around a certain pivot. What could be wrong with this? I'll attach a video to get a better perception of the issue in cause. I will also send the entire grabbing script just in case (I've set the physics handling as a comment so that it was easier to debug):

Code (CSharp):

1. using UnityEngine;
2.  
3. public class PlayerGrab : MonoBehaviour
4. {
5.     [Header("Grab")]
6.     public float reachDistance;
7.     public Transform grabPos;
8.     public Rigidbody grabPosRb;
9.     public Transform pivot;
10.     public Rigidbody pivotRb;
11.  
12.     public Collider grabObjectCollider;
13.  
14.     public float grabForce;
15.     public float grabDamping;
16.  
17.     public float grabAngularDrag;
18.  
19.     private bool inReach;
20.     private bool isGrabbing = false;
21.  
22.  
23.     [Header("Throw")]
24.     public float throwForce;
25.  
26.  
27.     [Header("Rotation")]
28.     public bool isRotating = false;
29.     public float rotatingSpeed;
30.  
31.     private Quaternion grabbedRotation; //Will use for later
32.  
33.     private float startAngularDrag;
34.  
35.  
36.     [Header("Scroll")]
37.     public float scrollForce;
38.  
39.     public float minScroll;
40.     public float maxScroll;
41.  
42.  
43.     [Header("Important")]
44.     public PlayerCam camScript;
45.     public PlayerMovement moveScript;
46.  
47.     public GameObject environment;
48.  
49.     public GameObject grabObject;
50.     private Rigidbody grabRb;
51.     public string objectTag;
52.  
53.     private RaycastHit hitInfo;
54.  
55.     private void Update()
56.     {
57.         inReach = Physics.Raycast(transform.position, camScript.transform.forward, out hitInfo, reachDistance);
58.  
59.         MyInput();
60.     }
61.  
62.     private void FixedUpdate()
63.     {
64.         if (isGrabbing)
65.         {
66.             //pivotRb.velocity += (grabPos.position - pivot.position) * grabForce * Time.deltaTime - pivotRb.velocity * grabDamping;
67.         }
68.  
69.         if (isRotating)
70.         {
71.             Rotate();
72.         }
73.     }
74.  
75.     private void MyInput()
76.     {
77.         if (Input.GetKeyDown(KeyCode.F))
78.         {
79.             pivot.position = hitInfo.point; //For debugging
80.         }
81.         if (Input.GetKeyDown(KeyCode.Mouse0))
82.         {
83.             CheckGrab();
84.         }
85.  
86.         if (Input.GetKeyDown(KeyCode.Mouse1) && isGrabbing)
87.         {
88.             Throw();
89.         }
90.  
91.         if (Input.GetKeyDown(KeyCode.R) && isGrabbing)
92.         {
93.             if (isRotating)
94.             {
95.                 grabbedRotation = pivot.rotation;
96.                 isRotating = false;
97.             }
98.             else
99.             {
100.                 isRotating = true;
101.             }
102.         }
103.  
104.         ScrollPosition(Input.GetAxis("Mouse ScrollWheel"));
105.     }
106.  
107.     private void ScrollPosition(float value)
108.     {
109.         grabPos.GetComponent<Rigidbody>().AddRelativeForce(new Vector3(0f, 0f, value * scrollForce), ForceMode.Force);
110.         grabPos.localPosition = new Vector3(0f, 0f, Mathf.Clamp(grabPos.localPosition.z, minScroll, maxScroll));
111.     }
112.  
113.     private void CheckGrab()
114.     {
115.         if (isGrabbing)
116.         {
117.             Release();
118.         }
119.         else if (hitInfo.rigidbody != null && hitInfo.collider.tag == objectTag && inReach)
120.         {
121.             Grab();
122.         }
123.     }
124.  
125.     private void Grab()
126.     {
127.  
128.         grabObject = hitInfo.collider.gameObject;
129.         grabRb = hitInfo.rigidbody;
130.  
131.         Debug.Log("Pivot position before: " + pivot.position);
132.         Debug.Log("Raycast hit point before: " + hitInfo.point);
133.  
134.         pivotRb.angularVelocity = grabRb.angularVelocity;
135.         pivot.rotation = grabObject.transform.rotation;
136.  
137.         grabRb.isKinematic = true;
138.         pivot.position = hitInfo.point;
139.         grabPos.position = hitInfo.point;
140.  
141.         grabObject.GetComponent<Collider>().enabled = false;
142.         //grabObjectCollider = hitInfo.collider.GetComponent<Collider>(); // Might give some trouble later oopsie doopsie :>
143.         //pivot.gameObject.AddComponent(grabObjectCollider.GetType());
144.  
145.         grabObject.transform.SetParent(pivot);
146.         GetColliderType(grabObject.transform.localPosition);
147.  
148.         Debug.Log("Pivot position after: " + pivot.position);
149.         Debug.Log("Raycast hit point after: " + hitInfo.point);
150.  
151.         isGrabbing = true;
152.     }
153.  
154.     private void GetColliderType(Vector3 position)
155.     {
156.         if (grabObjectCollider.GetType() == typeof(SphereCollider))
157.         {
158.             pivot.GetComponent<SphereCollider>().center = position;
159.             pivot.GetComponent<SphereCollider>().radius = Mathf.Max(grabObject.transform.localScale.x, grabObject.transform.localScale.y, grabObject.transform.localScale.z)/2f;
160.         }
161.         else if (grabObjectCollider.GetType() == typeof(BoxCollider))
162.         {
163.             pivot.GetComponent<BoxCollider>().center = position;
164.             pivot.GetComponent<BoxCollider>().size = grabObject.transform.localScale;
165.         }
166.         else if (grabObjectCollider.GetType() == typeof(CapsuleCollider))
167.         {
168.             pivot.GetComponent<CapsuleCollider>().center = position;
169.             pivot.GetComponent<CapsuleCollider>().height = 2f;
170.         }
171.     }
172.  
173.     private void Release()
174.     {
175.         Debug.Log(pivot.position);
176.         grabRb.isKinematic = false;
177.         grabRb.velocity += pivotRb.velocity;
178.         grabRb.angularVelocity += pivotRb.angularVelocity;
179.         pivotRb.velocity = Vector3.zero;
180.         pivotRb.angularVelocity = Vector3.zero;
181.         //Destroy(pivot.gameObject.GetComponent<Collider>());
182.         grabObject.GetComponent<Collider>().enabled = true;
183.  
184.         grabObject.transform.parent = environment.transform;
185.  
186.         isGrabbing = false;
187.         isRotating = false;
188.         grabObject = null;
189.         grabRb = null;
190.     }
191.  
192.     private void Throw()
193.     {
194.         grabRb.isKinematic = false;
195.         grabRb.AddForce(camScript.transform.forward.normalized * throwForce, ForceMode.Impulse);
196.         Release();
197.     }
198.  
199.     private void Rotate()
200.     {
201.         pivotRb.angularDrag = 10f;
202.         if (moveScript.verticalInput == 0f && moveScript.horizontalInput == 0f)
203.         {
204.             // torque towards last modified rotation
205.         }
206.         else
207.         {
208.             Vector3 horTorque = (-camScript.transform.up * moveScript.horizontalInput).normalized;
209.             Vector3 verTorque = (camScript.transform.right * moveScript.verticalInput).normalized;
210.             pivotRb.AddTorque((verTorque + horTorque) * rotatingSpeed, ForceMode.Acceleration);
211.  
212.             grabbedRotation = pivot.rotation;
213.         }
214.     }
215. }
216.  

Any help is very appreciated as I've been stuck in this over a week. Let me know if I need to provide any more info, thanks!