diff --git a/openapi_schema_validator/_format.py b/openapi_schema_validator/_format.py
index 6653b81..92b4f61 100644
--- a/openapi_schema_validator/_format.py
+++ b/openapi_schema_validator/_format.py
@@ -68,10 +68,13 @@ def is_byte(instance: object) -> bool:
     if not isinstance(instance, (str, bytes)):
         return True
     if isinstance(instance, str):
-        instance = instance.encode()
+        instance = instance.encode("ascii", errors="strict")
 
-    encoded = b64encode(b64decode(instance))
-    return encoded == instance
+    try:
+        b64decode(instance, validate=True)
+    except (binascii.Error, ValueError):
+        return False
+    return True
 
 
 def is_password(instance: object) -> bool:
diff --git a/tests/integration/test_validators.py b/tests/integration/test_validators.py
index 08f83ba..d9e928b 100644
--- a/tests/integration/test_validators.py
+++ b/tests/integration/test_validators.py
@@ -297,6 +297,10 @@ def test_nullable_enum_with_none(self, validator_class):
         "value",
         [
             b64encode(b"string").decode(),
+            b64encode(b"\x00\x01\x02").decode(),
+            "",
+            "AQ==",
+            "SGVsbG8=",
         ],
     )
     def test_string_format_byte_valid(self, validator_class, value):
@@ -309,7 +313,19 @@ def test_string_format_byte_valid(self, validator_class, value):
 
         assert result is None
 
-    @pytest.mark.parametrize("value", ["string"])
+    @pytest.mark.parametrize(
+        "value",
+        [
+            "string",
+            "SGVsbG8",
+            "SGVsbG8===",
+            "SGVsbG8$",
+            "SGVsbG8 ",
+            "SGVsbG8\n",
+            "SGVsbG8_",
+            "SGVsbG8-",
+        ],
+    )
     def test_string_format_byte_invalid(self, validator_class, value):
         schema = {"type": "string", "format": "byte"}
         validator = validator_class(