Sparkmax soft limit messes with motor speed

I am newish to programming and I am working on old code base. I wanted to implement some soft limits. They have code to do it but it was disabled when I first looked at it. I have stripped the code to the minimum required to move the arm and it works fine until I activate the soft limits.

    private void configureMotors() {
        lowerMotor.restoreFactoryDefaults();
        upperMotor.restoreFactoryDefaults();

        lowerMotor.enableSoftLimit(SoftLimitDirection.kForward, true);
        lowerMotor.enableSoftLimit(SoftLimitDirection.kReverse, true);
        lowerMotor.setSoftLimit(SoftLimitDirection.kForward, SoftStop.LOWER_FORWARD);
        lowerMotor.setSoftLimit(SoftLimitDirection.kReverse, SoftStop.LOWER_REVERSE);
        lowerMotor.setIdleMode(IdleMode.kBrake);
        lowerMotor.setSmartCurrentLimit(frc.robot.Constants.CurrentLimits.lowerArm);
        lowerMotor.setInverted(IsInverted.LOWER_MOTOR);

        upperMotor.enableSoftLimit(SoftLimitDirection.kForward, true);
        upperMotor.enableSoftLimit(SoftLimitDirection.kReverse, true);
        upperMotor.setSoftLimit(SoftLimitDirection.kForward, SoftStop.UPPER_FORWARD);
        upperMotor.setSoftLimit(SoftLimitDirection.kReverse, SoftStop.UPPER_REVERSE);
        upperMotor.setIdleMode(IdleMode.kBrake);
        upperMotor.setSmartCurrentLimit(CurrentLimits.upperArm);
        upperMotor.setInverted(IsInverted.UPPER_MOTOR);
    }