[fake_tensor] Don't run fallback for fbgemm ops (pytorch#106210)

Summary:
This diff also adds more warning messages around allowing a namespace into the
fallback. We need to grandfather in an operator to actually merge this diff.

Test Plan: - existing tests

Differential Revision: D47873841

Pull Request resolved: pytorch#106210
Approved by: https://github.com/eellison

torch/_subclasses/fake_tensor.py

@@ -1420,22 +1420,34 @@ def dispatch(self, func, types, args=(), kwargs=None):
                 if op_impl_out != NotImplemented:
                     return op_impl_out
 
-        def can_fallback(func: OpOverload):
+        def can_run_unsafe_fallback(func: OpOverload):
             if not self.allow_fallback_kernels:
                 return False
             # It's OK to try the fallback for built-in ops (e.g. aten, prims)
             # because we control and test these but the fallback leads to unexpected behavior
             # in user-defined custom ops
-            return func.namespace in {
+            #
+            # WARNING: DO NOT add any additional namespaces/operators here if they refer to operators
+            # outside of the pytorch/pytorch library! Any pre-existing things here
+            # are either in the pytorch/pytorch library or have been grandfathered in.
+            # The fallback does not always work and MAY CRASH and emit unreadable error messages
+            # so it should not be allowed by default.
+            allowed_namespaces = {
                 "debugprims",
                 "prims",
                 "aten",
                 "xla",
                 "vision",
                 "torchtext",
                 "torchaudio",
-                "fbgemm",
             }
+            grandfathered_ops_FIXME = {
+                "fbgemm::gmm",
+            }
+            return (
+                func.namespace in allowed_namespaces
+                or func.name() in grandfathered_ops_FIXME
+            )
 
         # run kernel registered to meta for func, which include
         # python meta registrations, prims, decomps, and c++ meta fns (structured kernels)
@@ -1444,7 +1456,7 @@ def can_fallback(func: OpOverload):
                 r = func(*args, **kwargs)
         except NotImplementedError as not_implemented_error:
             # no meta kernel registered, fallback to kernel for the device
-            if has_symbolic_sizes or not can_fallback(func):
+            if has_symbolic_sizes or not can_run_unsafe_fallback(func):
                 raise UnsupportedOperatorException(func)
             return run_fallback_kernel(self, func, args, kwargs, not_implemented_error)