how to Check Mouse direction while mouse is pressed?

I am creating a sliding block game and for that, I need a script that can take input from a keyboard as well as touch from a phone So that I can test easily. Till now I am able to create a script with that I am able to drag a block. It works perfectly with a mouse as well as on phone. But I want to add few things which I am not able to do.

1. While mouse pressed suppose I move the mouse right first but then I want to move the mouse left. Currently, it is only showing the starting direction of the mouse. It is because I am taking mouse Position when I press down and the I am subtracting that which is giving me the direction of the mouse. But I don't know how to check inside Input.GetMouseButton(0). So that I can check the direction while pressing down.
2. Hold and move the block. Suppose I have clicked on a block but didn't move for a few milliseconds but after that, I move left or right.

Code (CSharp):

1.     swipeThreshold is .01
2.  
3.      
4.  public enum WhichDirectionMouseIsMoving { LEFT, RIGHT, UP, DOWN, NONE }
5.     public WhichDirectionMouseIsMoving whichDirectionMouseIsMoving;
6.  
7.     void Update()
8.     {
9.         if (Input.GetMouseButtonDown(0))
10.         {
11.             previousPos = cam.ScreenToWorldPoint(Input.mousePosition);
12.             this.fingerDown = previousPos;
13.         }
14.         if (Input.GetMouseButton(0))
15.         {
16.  
17.             Vector2 pos = cam.ScreenToWorldPoint(Input.mousePosition);
18.  
19.             this.CheckMouseDirection((pos));
20.  
21.         }
22.         /// THIS IS for touch input
23.         foreach (Touch touch in Input.touches)
24.         {
25.             if (touch.phase == TouchPhase.Began)
26.             {
27.                 this.fingerDown = (touch.position);
28.                 previousPos = (this.fingerDown);
29.  
30.             }
31.             if (touch.phase == TouchPhase.Moved)
32.             {
33.                 this.fingerDown = touch.position;
34.                 this.CheckMouseDirection((this.fingerDown));
35.             }
36.         }
37.     }
38. }
39.  
40. //method responsible for check where mouse gone
41. private void CheckMouseDirection(Vector2 mousePos)
42. {
43.     if (moveVertical || moveHorizontal) return;
44.     //substract mouse previous pos with current pos
45.     float deltaX = this.fingerDown.x - mousePos.x;
46.     float deltaY = this.fingerDown.y - mousePos.y;
47.     //if greater than o.1 then we have move
48.     if ((Mathf.Abs(deltaX) > this.swipeThreshold)) { moveHorizontal = true; }
49.     else if ((Mathf.Abs(deltaY) > this.swipeThreshold)) { moveVertical = true; }
50.  
51.     if (moveHorizontal)
52.     {
53.         Debug.Log("checking horizontal");
54.         if (deltaX > 0)
55.         {
56.  
57.             whichDirectionMouseIsMoving = WhichDirectionMouseIsMoving.LEFT;
58.  
59.             MoveRightAndUp();
60.             Debug.Log("right");
61.  
62.         }
63.         else if (deltaX < 0)
64.         {
65.  
66.             whichDirectionMouseIsMoving = WhichDirectionMouseIsMoving.RIGHT;
67.  
68.             MoveRightAndUp();
69.             Debug.Log("Right");
70.         }
71.  
72.  
73.     }
74.     else if (moveVertical)
75.  
76.     {
77.         if (deltaY > 0)
78.         {
79.             whichDirectionMouseIsMoving = WhichDirectionMouseIsMoving.DOWN;
80.  
81.             MoveRightAndUp();
82.             //Debug.Log("down");
83.         }
84.         else if (deltaY < 0)
85.         {
86.             whichDirectionMouseIsMoving = WhichDirectionMouseIsMoving.UP;
87.  
88.             MoveRightAndUp();
89.             Debug.Log("up");
90.         }
91.  
92.     }
93. }
94.