torch finfo and iinfo - 文章 - 开发者社区

  
# docs/source/type_info.rst  
Type Info  
=========  
  
The numerical properties of a :class:`torch.dtype` can be accessed through either the :class:`torch.finfo` or the :class:`torch.iinfo`.  
  
.. _finfo-doc:  
  
torch.finfo  
-----------  
  
.. class:: torch.finfo  
  
A :class:`torch.finfo` is an object that represents the numerical properties of a floating point  
:class:`torch.dtype`, (i.e. ``torch.float32``, ``torch.float64``, ``torch.float16``, and ``torch.bfloat16``). This is similar to `numpy.finfo <https://docs.scipy.org/doc/numpy/reference/generated/numpy.finfo.html>`_.  
  
A :class:`torch.finfo` provides the following attributes:  
  
===============        =====   ==========================================================================  
Name                   Type    Description  
===============        =====   ==========================================================================  
bits                   int     The number of bits occupied by the type.  
eps                    float   The smallest representable number such that ``1.0 + eps != 1.0``.  
max                    float   The largest representable number.  
min                    float   The smallest representable number (typically ``-max``).  
tiny                   float   The smallest positive normal number. Equivalent to ``smallest_normal``.  
smallest_normal        float   The smallest positive normal number. See notes.  
resolution             float   The approximate decimal resolution of this type, i.e., ``10**-precision``.  
===============        =====   ==========================================================================  
  
.. note::  
  The constructor of :class:`torch.finfo` can be called without argument, in which case the class is created for the pytorch default dtype (as returned by :func:`torch.get_default_dtype`).  
  
.. note::  
  `smallest_normal` returns the smallest *normal* number, but there are smaller  
  subnormal numbers. See https://en.wikipedia.org/wiki/Denormal_number  
  for more information.  
  
  
torch.iinfo  
------------  
  
.. class:: torch.iinfo  
  
  
A :class:`torch.iinfo` is an object that represents the numerical properties of a integer  
:class:`torch.dtype` (i.e. ``torch.uint8``, ``torch.int8``, ``torch.int16``, ``torch.int32``, and ``torch.int64``). This is similar to `numpy.iinfo <https://docs.scipy.org/doc/numpy/reference/generated/numpy.iinfo.html>`_.  
  
A :class:`torch.iinfo` provides the following attributes:  
  
=========   =====   ========================================  
Name        Type    Description  
=========   =====   ========================================  
bits        int     The number of bits occupied by the type.  
max         int     The largest representable number.  
min         int     The smallest representable number.  
=========   =====   ========================================

finfo and iinfo

  
# Defined in torch/csrc/TypeInfo.cpp  
class iinfo:  
    bits: _int  
    min: _int  
    max: _int  
    dtype: str  
  
    def \_\_init\_\_(self, dtype: \_dtype) -> None: ...  
  
class finfo:  
    bits: _int  
    min: _float  
    max: _float  
    eps: _float  
    tiny: _float  
    smallest_normal: _float  
    resolution: _float  
    dtype: str  
  
    @overload  
    def \_\_init\_\_(self, dtype: \_dtype) -> None: ...  
  
    @overload  
    def \_\_init\_\_(self) -> None: ...

initModule

  
// torch/csrc/Module.cpp  
// separate decl and defn for msvc error C2491  
PyObject* initModule() {  
  ...  
  THPDTypeInfo_init(module);  
  ...  
}

THPDTypeInfo_init

  
// torch/csrc/TypeInfo.h  
struct THPDTypeInfo {  
  PyObject_HEAD at::ScalarType type;  
};  
  
struct THPFInfo : THPDTypeInfo {};  
  
struct THPIInfo : THPDTypeInfo {};  
  
extern PyTypeObject THPFInfoType;  
extern PyTypeObject THPIInfoType;  
  
inline bool THPFInfo\_Check(PyObject* obj) {  
  return Py_TYPE(obj) == &THPFInfoType  
}  
  
inline bool THPIInfo\_Check(PyObject* obj) {  
  return Py_TYPE(obj) == &THPIInfoType  
}  
  
void THPDTypeInfo\_init(PyObject* module);

ScalarType

  
// aten/src/ATen/core/ScalarType.h  
#include <c10/core/ScalarType.h>

  
// aten/src/ATen/ScalarType.h  
  
#pragma once  
#include <ATen/core/ATenGeneral.h> // for BC reasons  
#include <c10/core/Backend.h>  
#include <c10/core/ScalarType.h>

  
// c10/core/ScalarType.h  
  
// Scoped enumerations, ref: https://en.cppreference.com/w/cpp/language/enum  
enum class ScalarType : int8\_t {  
#define DEFINE\_ENUM(\_1, n) n,  
  AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_ENUM)  
#undef DEFINE\_ENUM  
      Undefined,  
  NumOptions  
};  
  
constexpr uint16\_t NumScalarTypes =  
    static\_cast<uint16\_t>(ScalarType::NumOptions);

Typeinfo

  
// torch/csrc/TypeInfo.cpp  
PyTypeObject THPFInfoType = {  
    PyVarObject_HEAD_INIT(nullptr, 0) "torch.finfo", /* tp\_name */  
    sizeof(THPFInfo),                                /* tp\_basicsize */  
    0,                                               /* tp\_itemsize */  
    nullptr,                                         /* tp\_dealloc */  
    0,                                               /* tp\_vectorcall\_offset */  
    nullptr,                                         /* tp\_getattr */  
    nullptr,                                         /* tp\_setattr */  
    nullptr,                                         /* tp\_reserved */  
    (reprfunc)THPFInfo_str,                          /* tp\_repr */  
    nullptr,                                         /* tp\_as\_number */  
    nullptr,                                         /* tp\_as\_sequence */  
    nullptr,                                         /* tp\_as\_mapping */  
    nullptr,                                         /* tp\_hash  */  
    nullptr,                                         /* tp\_call */  
    (reprfunc)THPFInfo_str,                          /* tp\_str */  
    nullptr,                                         /* tp\_getattro */  
    nullptr,                                         /* tp\_setattro */  
    nullptr,                                         /* tp\_as\_buffer */  
    Py_TPFLAGS_DEFAULT,                              /* tp\_flags */  
    nullptr,                                         /* tp\_doc */  
    nullptr,                                         /* tp\_traverse */  
    nullptr,                                         /* tp\_clear */  
    (richcmpfunc)THPDTypeInfo_compare,               /* tp\_richcompare */  
    0,                                               /* tp\_weaklistoffset */  
    nullptr,                                         /* tp\_iter */  
    nullptr,                                         /* tp\_iternext */  
    THPFInfo_methods,                                /* tp\_methods */  
    nullptr,                                         /* tp\_members */  
    THPFInfo_properties,                             /* tp\_getset */  
    nullptr,                                         /* tp\_base */  
    nullptr,                                         /* tp\_dict */  
    nullptr,                                         /* tp\_descr\_get */  
    nullptr,                                         /* tp\_descr\_set */  
    0,                                               /* tp\_dictoffset */  
    nullptr,                                         /* tp\_init */  
    nullptr,                                         /* tp\_alloc */  
    THPFInfo_pynew,                                  /* tp\_new */  
};

  
PyTypeObject THPIInfoType = {  
  ...  
};

THPObjectPtr and THPPointer

  
// torch/csrc/utils/object\_ptr.h  
  
/**  
 * An RAII-style, owning pointer to a PyObject.  You must protect  
 * destruction of this object with the GIL.  
 *  
 * WARNING: Think twice before putting this as a field in a C++  
 * struct.  This class does NOT take out the GIL on destruction,  
 * so if you will need to ensure that the destructor of your struct  
 * is either (a) always invoked when the GIL is taken or (b) takes  
 * out the GIL itself.  Easiest way to avoid this problem is to  
 * not use THPPointer in this situation.  
 */  
using THPObjectPtr = THPPointer<PyObject>;

  
// torch/csrc/utils/object\_ptr.h  
template<class T>  
class THPPointer {  
public:  
  THPPointer(): ptr(nullptr) {};  
  explicit THPPointer(T *ptr) noexcept : ptr(ptr) {};  
  THPPointer(THPPointer &&p) noexcept { free(); ptr = p.ptr; p.ptr = nullptr; };  
  
  ~THPPointer() { free(); };  
  T * get() { return ptr; }  
  const T * get() const { return ptr; }  
  T * release() { T *tmp = ptr; ptr = nullptr; return tmp; }  
  operator T*() { return ptr; }  
  THPPointer& operator =(T *new_ptr) noexcept { free(); ptr = new_ptr; return *this; }  
  THPPointer& operator =(THPPointer &&p) noexcept { free(); ptr = p.ptr; p.ptr = nullptr; return *this; }  
  T * operator ->() { return ptr; }  
  explicit operator bool() const { return ptr != nullptr; }  
  
private:  
  void free();  
  T *ptr = nullptr;  
};

THPFInfo_New

  
PyObject* THPFInfo\_New(const at::ScalarType& type) {  
  auto finfo = (PyTypeObject*)&THPFInfoType  
  auto self = THPObjectPtr{finfo->tp_alloc(finfo, 0)};  
  if (!self)  
    throw python_error();  
  auto self_ = reinterpret\_cast<THPDTypeInfo*>(self.get());  
  self_->type = c10::toRealValueType(type);  
  return self.release();  
}

toRealValueType

  
static inline ScalarType toRealValueType(ScalarType t) {  
  switch (t) {  
    case ScalarType::ComplexHalf:  
      return ScalarType::Half;  
    case ScalarType::ComplexFloat:  
      return ScalarType::Float;  
    case ScalarType::ComplexDouble:  
      return ScalarType::Double;  
    default:  
      return t;  
  }  
}

THPFInfo_pynew

  
PyObject* THPFInfo\_pynew(PyTypeObject* type, PyObject* args, PyObject* kwargs) {  
  HANDLE\_TH\_ERRORS  
  static torch::PythonArgParser parser({  
      "finfo(ScalarType type)",  
      "finfo()",  
  });  
  
  torch::ParsedArgs<1> parsed_args;  
  auto r = parser.parse(args, kwargs, parsed_args);  
  TORCH_CHECK(r.idx < 2, "Not a type");  
  at::ScalarType scalar_type;  
  if (r.idx == 1) {  
    scalar_type = torch::tensors::get_default_scalar_type();  
    // The default tensor type can only be set to a floating point type/  
    AT_ASSERT(at::isFloatingType(scalar_type));  
  } else {  
    scalar_type = r.scalartype(0);  
    if (!at::isFloatingType(scalar_type) && !at::isComplexType(scalar_type)) {  
      return PyErr_Format(  
          PyExc_TypeError,  
          "torch.finfo() requires a floating point input type. Use torch.iinfo to handle '%s'",  
          type->tp_name);  
    }  
  }  
  return THPFInfo_New(scalar_type);  
  END_HANDLE_TH_ERRORS  
}

THPFInfo_properties

  
static struct PyGetSetDef THPFInfo\_properties[] = {  
    {"bits", (getter)THPDTypeInfo_bits, nullptr, nullptr, nullptr},  
    {"eps", (getter)THPFInfo_eps, nullptr, nullptr, nullptr},  
    {"max", (getter)THPFInfo_max, nullptr, nullptr, nullptr},  
    {"min", (getter)THPFInfo_min, nullptr, nullptr, nullptr},  
    {"smallest\_normal", (getter)THPFInfo_smallest_normal, nullptr, nullptr, nullptr},  
    {"tiny", (getter)THPFInfo_tiny, nullptr, nullptr, nullptr},  
    {"resolution", (getter)THPFInfo_resolution, nullptr, nullptr, nullptr},  
    {"dtype", (getter)THPFInfo_dtype, nullptr, nullptr, nullptr},  
    {nullptr}};

  
  
static PyObject* THPDTypeInfo\_bits(THPDTypeInfo* self, void*) {  
  // NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions,cppcoreguidelines-avoid-magic-numbers)  
  int bits = elementSize(self->type) * 8;  
  return THPUtils_packInt64(bits);  
}  
  
static PyObject* THPFInfo\_eps(THPFInfo* self, void*) {  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16,  
      self->type, "epsilon", [] {  
        return PyFloat_FromDouble(  
            std::numeric_limits<  
                at::scalar_value_type<scalar\_t>::type>::epsilon());  
      });  
}  
  
static PyObject* THPFInfo\_max(THPFInfo* self, void*) {  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16, self->type, "max", [] {  
    return PyFloat_FromDouble(  
        std::numeric_limits<at::scalar_value_type<scalar\_t>::type>::max());  
  });  
}  
  
// std::numeric\_limits<T>::lowest  
// Returns the lowest finite value representable by the numeric type T, that is, a finite value x such that there is no other finite value y where y < x. This is different from `std::numeric\_limits<T>::min()` for floating-point types. Only meaningful for bounded types.  
static PyObject* THPFInfo\_min(THPFInfo* self, void*) {  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16, self->type, "lowest", [] {  
    return PyFloat_FromDouble(  
        std::numeric_limits<at::scalar_value_type<scalar\_t>::type>::lowest());  
  });  
}  
  
// std::numeric\_limits<T>::min  
// Returns the minimum finite value representable by the numeric type T.  
// For floating-point types with denormalization, min() returns the minimum positive normalized value. Note that this behavior may be unexpected, especially when compared to the behavior of min() for integral types. To find the value that has no values less than it, use `lowest()``.   
static PyObject* THPFInfo\_smallest\_normal(THPFInfo* self, void*) {  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16, self->type, "min", [] {  
    return PyFloat_FromDouble(  
        std::numeric_limits<at::scalar_value_type<scalar\_t>::type>::min());  
  });  
}  
  
static PyObject* THPFInfo\_tiny(THPFInfo* self, void*) {  
  // see gh-70909, essentially the array\_api prefers smallest\_normal over tiny  
  return THPFInfo_smallest_normal(self, nullptr);  
}  
  
static PyObject* THPFInfo\_resolution(THPFInfo* self, void*) {  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16, self->type, "digits10", [] {  
    return PyFloat_FromDouble(  
        std::pow(10, -std::numeric_limits<at::scalar_value_type<scalar\_t>::type>::digits10));  
  });  
}  
  
static PyObject* THPFInfo\_dtype(THPFInfo* self, void*) {  
  std::string primary_name, legacy_name;  
  std::tie(primary_name, legacy_name) = torch::utils::getDtypeNames(self->type);  
  // NOLINTNEXTLINE(clang-diagnostic-unused-local-typedef)  
  return AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(at::kHalf, at::ScalarType::BFloat16, self->type, "dtype", [primary_name] {  
    return PyUnicode_FromString((char*)primary_name.data());  
  });  
}

THPFInfo_methods

  
static PyMethodDef THPFInfo_methods[] = {  
    {nullptr} /* Sentinel */  
};

THPDTypeInfo_compare

  
PyObject* THPDTypeInfo\_compare(THPDTypeInfo* a, THPDTypeInfo* b, int op) {  
  switch (op) {  
    case Py_EQ:  
      if (a->type == b->type) {  
        Py_RETURN_TRUE;  
      } else {  
        Py_RETURN_FALSE;  
      }  
    case Py_NE:  
      if (a->type != b->type) {  
        Py_RETURN_TRUE;  
      } else {  
        Py_RETURN_FALSE;  
      }  
  }  
  return Py_INCREF(Py_NotImplemented), Py_NotImplemented;  
}

THPFInfo_str

  
PyObject* THPFInfo\_str(THPFInfo* self) {  
  std::ostringstream oss;  
  oss << "finfo(resolution=" << PyFloat_AsDouble(THPFInfo_resolution(self, nullptr));  
  oss << ", min=" << PyFloat_AsDouble(THPFInfo_min(self, nullptr));  
  oss << ", max=" << PyFloat_AsDouble(THPFInfo_max(self, nullptr));  
  oss << ", eps=" << PyFloat_AsDouble(THPFInfo_eps(self, nullptr));  
  oss << ", smallest\_normal=" << PyFloat_AsDouble(THPFInfo_smallest_normal(self, nullptr));  
  oss << ", tiny=" << PyFloat_AsDouble(THPFInfo_tiny(self, nullptr));  
  oss << ", dtype=" << PyUnicode_AsUTF8(THPFInfo_dtype(self, nullptr)) << ")";  
  
  return THPUtils_packString(oss.str().c_str());  
}

参考文献

https://en.cppreference.com/w/cpp/language/enum

picture.image