Unity rigidbody collision not working properly

Im having a lot of weird prolems with my rigidbody collision in unity. I have a player controller script which allows me to move my player, but when it collides with stairs, it glitches out. when it collides with a door, it glitches out and when it collides with 2 nvisible box colliders at the edge of the map while walking diagonally, it walks through one of them. Ive searched around quite a bit but couldn't find anything. I know it's not a lot to go on, but here is some stuff that might help:

a video of what happens exactly
the character controller code:

    using System;
    using UnityEngine;

public class PlayerController : MonoBehaviour{
    public Rigidbody body;

    //player movement
    private float speed = 12f;
    private float walkSpeed = 10;
    private float runSpeed = 15;
    private float gravity = -9.81f;
    public float jumpHeight = 2f;
    private Vector3 inputs;

    //player rotation
    private float targetAngle = 0f;
    private float angle = 0f;
    public float turnSmoothTime = .1f;
    public float turnSmoothVelocity;

    //player jump
    public Transform groundCheck;
    public float groundDistance = 0.4f;
    public LayerMask groundMask;
    private bool isGrounded;

    //there are 6 possible directions for gravity; positive and negative x, y and z. The direction can therefore be -3, -2, -1, 1, 2 or 3 where 1=y, 2=x, 3=z
    public int direction = 1;

    public void movePlayer(Vector2 movement){

        float horizontal = movement.x;
        float vertical = movement.y;
        
        //check if the player is standing on the ground
        isGrounded = Physics.CheckSphere(groundCheck.position, groundDistance, groundMask);

        Quaternion rotation = new Quaternion();

        body.freezeRotation = true;

        if (Mathf.Abs(direction) == 1){
            //gravity in y direction
            //set the direction of the gravity
            Physics.gravity = new Vector3(0f, direction * gravity, 0f);

            //set the direction the inputs should work in
            inputs.x = horizontal;
            inputs.z = vertical;
            inputs.y = body.velocity.y;

            //calculate the angle with which the player has to be rotated and make the rotation smooth (smoothing is only possible in this orientation)
            targetAngle = Mathf.Atan2(inputs.x, inputs.z) * Mathf.Rad2Deg;
            angle = Mathf.SmoothDampAngle(transform.eulerAngles.y, targetAngle, ref turnSmoothVelocity, turnSmoothTime);

            //set the characters rotation
            rotation = Quaternion.Euler(0f, angle, 0f);
        }
        else if (Mathf.Abs(direction) == 2){
            //gravity in x direction
            Physics.gravity = new Vector3((direction / 2) * gravity, 0f, 0f);

            inputs.y = -horizontal;
            inputs.z = vertical;
            inputs.x = body.velocity.x;
            
            targetAngle = Mathf.Atan2(-inputs.y, inputs.z) * Mathf.Rad2Deg;

            rotation = Quaternion.Euler(targetAngle, 0f, (direction / 2) * -90f);
        }
        else if (Mathf.Abs(direction) == 3){
            //gravity in z-direction

            Physics.gravity = new Vector3(0f, 0f, (direction / 3) * -gravity);

            inputs.x = horizontal;
            inputs.y = vertical;
            inputs.z = body.velocity.z;
            
            targetAngle = Mathf.Atan2(inputs.x, inputs.y) * Mathf.Rad2Deg;

            //set the rotation in the correct order of the axis (90 degrees first and then around the correct axis)
            rotation = Quaternion.AngleAxis((direction / 3) * -90f, Vector3.right) *
                       Quaternion.AngleAxis(0f, Vector3.forward) *
                       Quaternion.AngleAxis(targetAngle, Vector3.up);
        }
        else{

            direction = 1;
        }

        /*
        if (inputs != Vector3.zero){
            body.velocity = inputs;
        }*/
        
        //rotate the player in the move direction as long as they are moving
        if (inputs.magnitude >= 0.1f){
            transform.rotation = rotation;
        }
        
    }

    void FixedUpdate(){
//move the player
        body.MovePosition(body.position + inputs * speed * Time.fixedDeltaTime);
    }

    public void flip(int changedDirection){
        inputs = Vector3.zero;
        angle = 0f;
        targetAngle = 0f;
        direction = changedDirection;
    }

    public void walk(){
        if (isGrounded){
            speed = walkSpeed;
        }
    }

    public void run(){
        if (isGrounded){
            speed = runSpeed;
        }
    }

    public void jump(){
        if (isGrounded){
            if (direction == 1){
                body.velocity = new Vector3(inputs.x, jumpHeight, inputs.z);
            }
            else if (direction == 2){
                body.velocity = new Vector3(jumpHeight, inputs.y, inputs.z);
            }
            else if (direction == 3){
                body.velocity = new Vector3(inputs.x, inputs.y, jumpHeight);
            }
        }
    }
}

A screenshot of my player object's important components:

P.S. the code has some weird gravty changing parts. Thats for the rest of the game, but it's not important for this question. Hopefully you guys can help me. Im happy to provide any other information you might need :) thanks for your time in advance!!

Edit: Commented out one part based on the first answer, but that din't fix it

Solution

Okay so for anyone having similar problems, I found the answer:

The problem was that I was using body.moveposition, this should actually body.addforce.

This oes mean that you have to change some of the code. If you want further explanation because you had a similar problem, contact me, but here is my improved code to get you started:

using System;
using UnityEngine;

public class PlayerController : MonoBehaviour{
    public Rigidbody body;

    //player movement
    private float speed = 12f;
    private float walkSpeed = 10;
    private float runSpeed = 15;
    public float gravity = -9.81f;
    public float jumpHeight = 2f;
    private Vector3 inputs;

    //player rotation
    private float targetAngle = 0f;
    private float angle = 0f;
    public float turnSmoothTime = .1f;
    public float turnSmoothVelocity;

    //player jump
    public Transform groundCheck;
    public float groundDistance = 0.4f;
    public LayerMask groundMask;
    private bool isGrounded;

    //there are 6 possible directions for gravity; positive and negative x, y and z. The direction can therefore be -3, -2, -1, 1, 2 or 3 where 1=y, 2=x, 3=z
    public int direction = 1;

    public void movePlayer(Vector2 movement){

        float horizontal = movement.x;
        float vertical = movement.y;
        
        //check if the player is standing on the ground
        isGrounded = Physics.CheckSphere(groundCheck.position, groundDistance, groundMask);

        Quaternion rotation = new Quaternion();

        body.freezeRotation = true;

        if (Mathf.Abs(direction) == 1){
            //gravity in y direction
            //set the direction of the gravity
            Physics.gravity = new Vector3(0f, direction * gravity, 0f);

            //set the direction the inputs should work in
            inputs.x = horizontal;
            inputs.z = vertical;
            inputs.y = 0;

            //calculate the angle with which the player has to be rotated and make the rotation smooth (smoothing is only possible in this orientation)
            targetAngle = Mathf.Atan2(inputs.x, inputs.z) * Mathf.Rad2Deg;
            angle = Mathf.SmoothDampAngle(transform.eulerAngles.y, targetAngle, ref turnSmoothVelocity, turnSmoothTime);

            //set the characters rotation
            rotation = Quaternion.Euler(0f, angle, 0f);
        }
        else if (Mathf.Abs(direction) == 2){
            //gravity in x direction
            Physics.gravity = new Vector3((direction / 2) * gravity, 0f, 0f);

            inputs.y = -horizontal;
            inputs.z = vertical;
            inputs.x = body.velocity.x;
            
            targetAngle = Mathf.Atan2(-inputs.y, inputs.z) * Mathf.Rad2Deg;

            rotation = Quaternion.Euler(targetAngle, 0f, (direction / 2) * -90f);
        }
        else if (Mathf.Abs(direction) == 3){
            //gravity in z-direction

            Physics.gravity = new Vector3(0f, 0f, (direction / 3) * -gravity);

            inputs.x = horizontal;
            inputs.y = vertical;
            inputs.z = body.velocity.z;
            
            targetAngle = Mathf.Atan2(inputs.x, inputs.y) * Mathf.Rad2Deg;

            //set the rotation in the correct order of the axis (90 degrees first and then around the correct axis)
            rotation = Quaternion.AngleAxis((direction / 3) * -90f, Vector3.right) *
                       Quaternion.AngleAxis(0f, Vector3.forward) *
                       Quaternion.AngleAxis(targetAngle, Vector3.up);
        }
        else{

            direction = 1;
        }

        //rotate the player in the move direction as long as they are moving
        if (inputs.magnitude >= 0.1f){
            transform.rotation = rotation;
        }
        
    }

    void FixedUpdate(){
        body.AddForce(inputs * speed * Time.fixedDeltaTime);
    }

    public void flip(int changedDirection){
        inputs = Vector3.zero;
        angle = 0f;
        targetAngle = 0f;
        direction = changedDirection;
    }

    public void walk(){
        if (isGrounded){
            speed = walkSpeed;
        }
    }

    public void run(){
        if (isGrounded){
            speed = runSpeed;
        }
    }

    public void jump(){
        if (isGrounded){
            if (direction == 1){
                body.AddForce(new Vector3(0, jumpHeight, 0));
            }
            else if (direction == 2){
                body.AddForce(new Vector3(jumpHeight, 0, 0));
            }
            else if (direction == 3){
                body.AddForce(new Vector3(0,0 , jumpHeight));
            }
        }
    }
}

For me this created some additionl problems with jump for example, so here are the values you need to tweak to fix those: the rigidbody mass and drag and the player controller gravity and jump height.