Lines Matching refs:uint64_t
74 static inline uint64_t checkedMul(uint64_t lhs, uint64_t rhs) { in checkedMul()
75 assert((lhs == 0 || rhs <= std::numeric_limits<uint64_t>::max() / lhs) && in checkedMul()
103 assertPermutedSizesMatchShape(const std::vector<uint64_t> &dimSizes, in assertPermutedSizesMatchShape()
104 uint64_t rank, const uint64_t *perm, in assertPermutedSizesMatchShape()
105 const uint64_t *shape) { in assertPermutedSizesMatchShape()
108 for (uint64_t r = 0; r < rank; r++) in assertPermutedSizesMatchShape()
123 Element(uint64_t *ind, V val) : indices(ind), value(val){}; in Element()
124 uint64_t *indices; // pointer into shared index pool
133 const std::function<void(const std::vector<uint64_t> &, V)> &;
143 SparseTensorCOO(const std::vector<uint64_t> &dimSizes, uint64_t capacity) in SparseTensorCOO()
152 void add(const std::vector<uint64_t> &ind, V val) { in add()
154 uint64_t *base = indices.data(); in add()
155 uint64_t size = indices.size(); in add()
156 uint64_t rank = getRank(); in add()
158 for (uint64_t r = 0; r < rank; r++) { in add()
167 uint64_t *newBase = indices.data(); in add()
169 for (uint64_t i = 0, n = elements.size(); i < n; i++) in add()
182 uint64_t rank = getRank(); in sort()
185 for (uint64_t r = 0; r < rank; r++) { in sort()
195 uint64_t getRank() const { return dimSizes.size(); } in getRank()
198 const std::vector<uint64_t> &getDimSizes() const { return dimSizes; } in getDimSizes()
224 static SparseTensorCOO<V> *newSparseTensorCOO(uint64_t rank, in newSparseTensorCOO()
225 const uint64_t *dimSizes, in newSparseTensorCOO()
226 const uint64_t *perm, in newSparseTensorCOO()
227 uint64_t capacity = 0) { in newSparseTensorCOO()
228 std::vector<uint64_t> permsz(rank); in newSparseTensorCOO()
229 for (uint64_t r = 0; r < rank; r++) { in newSparseTensorCOO()
237 const std::vector<uint64_t> dimSizes; // per-dimension sizes
239 std::vector<uint64_t> indices; // shared index pool
266 SparseTensorStorageBase(const std::vector<uint64_t> &dimSizes, in SparseTensorStorageBase()
267 const uint64_t *perm, const DimLevelType *sparsity) in SparseTensorStorageBase()
271 const uint64_t rank = getRank(); in SparseTensorStorageBase()
274 for (uint64_t r = 0; r < rank; r++) { in SparseTensorStorageBase()
281 for (uint64_t r = 0; r < rank; r++) in SparseTensorStorageBase()
288 uint64_t getRank() const { return dimSizes.size(); } in getRank()
291 const std::vector<uint64_t> &getDimSizes() const { return dimSizes; } in getDimSizes()
294 uint64_t getDimSize(uint64_t d) const { in getDimSize()
301 const std::vector<uint64_t> &getRev() const { return rev; } in getRev()
307 bool isCompressedDim(uint64_t d) const { in isCompressedDim()
314 virtual void newEnumerator(SparseTensorEnumeratorBase<V> **, uint64_t, \
315 const uint64_t *) const { \
323 virtual void getPointers(std::vector<P> **, uint64_t) { \
329 virtual void getIndices(std::vector<I> **, uint64_t) { \
343 virtual void lexInsert(const uint64_t *, V) { FATAL_PIV("lexInsert" #VNAME); }
349 virtual void expInsert(uint64_t *, V *, bool *, uint64_t *, uint64_t) { \
369 const std::vector<uint64_t> dimSizes;
370 std::vector<uint64_t> rev;
394 SparseTensorStorage(const std::vector<uint64_t> &dimSizes, in SparseTensorStorage()
395 const uint64_t *perm, const DimLevelType *sparsity) in SparseTensorStorage()
405 SparseTensorStorage(const std::vector<uint64_t> &dimSizes, in SparseTensorStorage()
406 const uint64_t *perm, const DimLevelType *sparsity, in SparseTensorStorage()
415 uint64_t sz = 1; in SparseTensorStorage()
416 for (uint64_t r = 0, rank = getRank(); r < rank; r++) { in SparseTensorStorage()
436 uint64_t nnz = elements.size(); in SparseTensorStorage()
451 SparseTensorStorage(const std::vector<uint64_t> &dimSizes,
452 const uint64_t *perm, const DimLevelType *sparsity,
458 void getPointers(std::vector<P> **out, uint64_t d) final { in getPointers()
462 void getIndices(std::vector<I> **out, uint64_t d) final { in getIndices()
469 void lexInsert(const uint64_t *cursor, V val) final { in lexInsert()
471 uint64_t diff = 0; in lexInsert()
472 uint64_t top = 0; in lexInsert()
485 void expInsert(uint64_t *cursor, V *values, bool *filled, uint64_t *added, in expInsert()
486 uint64_t count) final { in expInsert()
492 const uint64_t lastDim = getRank() - 1; in expInsert()
493 uint64_t index = added[0]; in expInsert()
500 for (uint64_t i = 1; i < count; i++) { in expInsert()
519 void newEnumerator(SparseTensorEnumeratorBase<V> **out, uint64_t rank, in newEnumerator()
520 const uint64_t *perm) const final { in newEnumerator()
528 SparseTensorCOO<V> *toCOO(const uint64_t *perm) const { in toCOO()
533 enumerator->forallElements([&coo](const std::vector<uint64_t> &ind, V val) { in toCOO()
552 newSparseTensor(uint64_t rank, const uint64_t *shape, const uint64_t *perm, in newSparseTensor()
560 std::vector<uint64_t> permsz(rank); in newSparseTensor()
561 for (uint64_t r = 0; r < rank; r++) { in newSparseTensor()
579 newSparseTensor(uint64_t rank, const uint64_t *shape, const uint64_t *perm, in newSparseTensor()
598 void appendPointer(uint64_t d, uint64_t pos, uint64_t count = 1) { in appendPointer()
614 void appendIndex(uint64_t d, uint64_t full, uint64_t i) { in appendIndex()
634 void writeIndex(uint64_t d, uint64_t pos, uint64_t i) { in writeIndex()
653 uint64_t assembledSize(uint64_t parentSz, uint64_t d) const { in assembledSize()
668 void fromCOO(const std::vector<Element<V>> &elements, uint64_t lo, in fromCOO()
669 uint64_t hi, uint64_t d) { in fromCOO()
670 uint64_t rank = getRank(); in fromCOO()
679 uint64_t full = 0; in fromCOO()
682 uint64_t i = elements[lo].indices[d]; in fromCOO()
683 uint64_t seg = lo + 1; in fromCOO()
698 void finalizeSegment(uint64_t d, uint64_t full = 0, uint64_t count = 1) { in finalizeSegment()
704 const uint64_t sz = getDimSizes()[d]; in finalizeSegment()
719 void endPath(uint64_t diff) { in endPath()
720 uint64_t rank = getRank(); in endPath()
722 for (uint64_t i = 0; i < rank - diff; i++) { in endPath()
723 const uint64_t d = rank - i - 1; in endPath()
729 void insPath(const uint64_t *cursor, uint64_t diff, uint64_t top, V val) { in insPath()
730 uint64_t rank = getRank(); in insPath()
732 for (uint64_t d = diff; d < rank; d++) { in insPath()
733 uint64_t i = cursor[d]; in insPath()
742 uint64_t lexDiff(const uint64_t *cursor) const { in lexDiff()
743 for (uint64_t r = 0, rank = getRank(); r < rank; r++) in lexDiff()
760 std::vector<uint64_t> idx; // index cursor for lexicographic insertion.
793 uint64_t rank, const uint64_t *perm) in SparseTensorEnumeratorBase()
800 for (uint64_t s = 0; s < rank; s++) { // `s` source storage-order in SparseTensorEnumeratorBase()
801 uint64_t t = perm[rev[s]]; // `t` target-order in SparseTensorEnumeratorBase()
819 uint64_t getRank() const { return permsz.size(); } in getRank()
822 const std::vector<uint64_t> &permutedSizes() const { return permsz; } in permutedSizes()
833 std::vector<uint64_t> permsz; // in target order.
834 std::vector<uint64_t> reord; // source storage-order -> target order.
835 std::vector<uint64_t> cursor; // in target order.
848 uint64_t rank, const uint64_t *perm) in SparseTensorEnumerator()
859 void forallElements(ElementConsumer<V> yield, uint64_t parentPos, in forallElements()
860 uint64_t d) { in forallElements()
875 const uint64_t pstart = static_cast<uint64_t>(pointersD[parentPos]); in forallElements()
876 const uint64_t pstop = static_cast<uint64_t>(pointersD[parentPos + 1]); in forallElements()
880 uint64_t &cursorReordD = this->cursor[this->reord[d]]; in forallElements()
881 for (uint64_t pos = pstart; pos < pstop; pos++) { in forallElements()
882 cursorReordD = static_cast<uint64_t>(indicesD[pos]); in forallElements()
886 const uint64_t sz = src.getDimSizes()[d]; in forallElements()
887 const uint64_t pstart = parentPos * sz; in forallElements()
888 uint64_t &cursorReordD = this->cursor[this->reord[d]]; in forallElements()
889 for (uint64_t i = 0; i < sz; i++) { in forallElements()
912 SparseTensorNNZ(const std::vector<uint64_t> &dimSizes, in SparseTensorNNZ()
918 uint64_t sz = 1; // the product of all `dimSizes` strictly less than `r`. in SparseTensorNNZ()
919 for (uint64_t rank = getRank(), r = 0; r < rank; r++) { in SparseTensorNNZ()
947 uint64_t getRank() const { return dimSizes.size(); } in getRank()
957 [this](const std::vector<uint64_t> &ind, V) { add(ind); }); in initialize()
961 using NNZConsumer = const std::function<void(uint64_t)> &;
967 void forallIndices(uint64_t stopDim, NNZConsumer yield) const { in forallIndices()
980 void add(const std::vector<uint64_t> &ind) { in add()
981 uint64_t parentPos = 0; in add()
982 for (uint64_t rank = getRank(), r = 0; r < rank; r++) { in add()
990 void forallIndices(NNZConsumer yield, uint64_t stopDim, uint64_t parentPos, in forallIndices()
991 uint64_t d) const { in forallIndices()
997 const uint64_t sz = dimSizes[d]; in forallIndices()
998 const uint64_t pstart = parentPos * sz; in forallIndices()
999 for (uint64_t i = 0; i < sz; i++) in forallIndices()
1005 const std::vector<uint64_t> &dimSizes;
1007 std::vector<std::vector<uint64_t>> nnz;
1012 const std::vector<uint64_t> &dimSizes, const uint64_t *perm, in SparseTensorStorage()
1022 uint64_t parentSz = 1; // assembled-size (not dimension-size) of `r-1`. in SparseTensorStorage()
1023 for (uint64_t rank = getRank(), r = 0; r < rank; r++) { in SparseTensorStorage()
1027 uint64_t currentPos = 0; in SparseTensorStorage()
1028 nnz.forallIndices(r, [this, ¤tPos, r](uint64_t n) { in SparseTensorStorage()
1052 enumerator->forallElements([this](const std::vector<uint64_t> &ind, V val) { in SparseTensorStorage()
1053 uint64_t parentSz = 1, parentPos = 0; in SparseTensorStorage()
1054 for (uint64_t rank = getRank(), r = 0; r < rank; r++) { in SparseTensorStorage()
1061 const uint64_t currentPos = pointers[r][parentPos]; in SparseTensorStorage()
1080 for (uint64_t parentSz = 1, rank = getRank(), r = 0; r < rank; r++) { in SparseTensorStorage()
1088 for (uint64_t n = 0; n < parentSz; n++) { in SparseTensorStorage()
1089 const uint64_t parentPos = parentSz - n; in SparseTensorStorage()
1193 uint64_t getRank() const { in getRank()
1199 uint64_t getNNZ() const { in getNNZ()
1207 const uint64_t *getDimSizes() const { return idata + 2; } in getDimSizes()
1211 uint64_t getDimSize(uint64_t d) const { in getDimSize()
1218 void assertMatchesShape(uint64_t rank, const uint64_t *shape) const { in assertMatchesShape()
1220 for (uint64_t r = 0; r < rank; r++) in assertMatchesShape()
1233 uint64_t idata[512];
1296 for (uint64_t r = 0; r < idata[0]; r++) in readExtFROSTTHeader()
1308 const std::vector<uint64_t> indices, in addValue()
1323 const std::vector<uint64_t> indices, in readCOOValue()
1343 const std::vector<uint64_t> indices, in readCOOValue()
1357 openSparseTensorCOO(char *filename, uint64_t rank, const uint64_t *shape, in openSparseTensorCOO()
1358 const uint64_t *perm, PrimaryType valTp) { in openSparseTensorCOO()
1375 uint64_t nnz = stfile.getNNZ(); in openSparseTensorCOO()
1379 std::vector<uint64_t> indices(rank); in openSparseTensorCOO()
1380 for (uint64_t k = 0; k < nnz; k++) { in openSparseTensorCOO()
1382 for (uint64_t r = 0; r < rank; r++) { in openSparseTensorCOO()
1383 uint64_t idx = strtoul(linePtr, &linePtr, 10); in openSparseTensorCOO()
1405 uint64_t rank = coo->getRank(); in outSparseTensor()
1406 uint64_t nnz = elements.size(); in outSparseTensor()
1411 for (uint64_t r = 0; r < rank - 1; r++) in outSparseTensor()
1414 for (uint64_t i = 0; i < nnz; i++) { in outSparseTensor()
1416 for (uint64_t r = 0; r < rank; r++) in outSparseTensor()
1427 static SparseTensorStorage<uint64_t, uint64_t, V> *
1428 toMLIRSparseTensor(uint64_t rank, uint64_t nse, uint64_t *shape, V *values, in toMLIRSparseTensor()
1429 uint64_t *indices, uint64_t *perm, uint8_t *sparse) { in toMLIRSparseTensor()
1433 std::vector<uint64_t> order(perm, perm + rank); in toMLIRSparseTensor()
1435 for (uint64_t i = 0; i < rank; ++i) in toMLIRSparseTensor()
1440 for (uint64_t i = 0; i < rank; ++i) in toMLIRSparseTensor()
1448 std::vector<uint64_t> idx(rank); in toMLIRSparseTensor()
1449 for (uint64_t i = 0, base = 0; i < nse; i++) { in toMLIRSparseTensor()
1450 for (uint64_t r = 0; r < rank; r++) in toMLIRSparseTensor()
1456 auto *tensor = SparseTensorStorage<uint64_t, uint64_t, V>::newSparseTensor( in toMLIRSparseTensor()
1464 static void fromMLIRSparseTensor(void *tensor, uint64_t *pRank, uint64_t *pNse, in fromMLIRSparseTensor()
1465 uint64_t **pShape, V **pValues, in fromMLIRSparseTensor()
1466 uint64_t **pIndices) { in fromMLIRSparseTensor()
1469 static_cast<SparseTensorStorage<uint64_t, uint64_t, V> *>(tensor); in fromMLIRSparseTensor()
1470 uint64_t rank = sparseTensor->getRank(); in fromMLIRSparseTensor()
1471 std::vector<uint64_t> perm(rank); in fromMLIRSparseTensor()
1476 uint64_t nse = elements.size(); in fromMLIRSparseTensor()
1478 uint64_t *shape = new uint64_t[rank]; in fromMLIRSparseTensor()
1479 for (uint64_t i = 0; i < rank; i++) in fromMLIRSparseTensor()
1483 uint64_t *indices = new uint64_t[rank * nse]; in fromMLIRSparseTensor()
1485 for (uint64_t i = 0, base = 0; i < nse; i++) { in fromMLIRSparseTensor()
1487 for (uint64_t j = 0; j < rank; j++) in fromMLIRSparseTensor()
1550 static_assert(std::is_same<index_type, uint64_t>::value,
1566 uint64_t rank = aref->sizes[0]; in _mlir_ciface_newSparseTensor()
1576 CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF64, uint64_t, in _mlir_ciface_newSparseTensor()
1577 uint64_t, double); in _mlir_ciface_newSparseTensor()
1578 CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF64, uint64_t, in _mlir_ciface_newSparseTensor()
1580 CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF64, uint64_t, in _mlir_ciface_newSparseTensor()
1582 CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF64, uint64_t, in _mlir_ciface_newSparseTensor()
1585 uint64_t, double); in _mlir_ciface_newSparseTensor()
1593 uint64_t, double); in _mlir_ciface_newSparseTensor()
1601 uint64_t, double); in _mlir_ciface_newSparseTensor()
1610 CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF32, uint64_t, in _mlir_ciface_newSparseTensor()
1611 uint64_t, float); in _mlir_ciface_newSparseTensor()
1612 CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF32, uint64_t, in _mlir_ciface_newSparseTensor()
1614 CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF32, uint64_t, in _mlir_ciface_newSparseTensor()
1616 CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF32, uint64_t, in _mlir_ciface_newSparseTensor()
1619 uint64_t, float); in _mlir_ciface_newSparseTensor()
1627 uint64_t, float); in _mlir_ciface_newSparseTensor()
1635 uint64_t, float); in _mlir_ciface_newSparseTensor()
1644 CASE_SECSAME(OverheadType::kU64, PrimaryType::kF16, uint64_t, f16); in _mlir_ciface_newSparseTensor()
1645 CASE_SECSAME(OverheadType::kU64, PrimaryType::kBF16, uint64_t, bf16); in _mlir_ciface_newSparseTensor()
1654 CASE_SECSAME(OverheadType::kU64, PrimaryType::kI64, uint64_t, int64_t); in _mlir_ciface_newSparseTensor()
1655 CASE_SECSAME(OverheadType::kU64, PrimaryType::kI32, uint64_t, int32_t); in _mlir_ciface_newSparseTensor()
1656 CASE_SECSAME(OverheadType::kU64, PrimaryType::kI16, uint64_t, int16_t); in _mlir_ciface_newSparseTensor()
1657 CASE_SECSAME(OverheadType::kU64, PrimaryType::kI8, uint64_t, int8_t); in _mlir_ciface_newSparseTensor()
1672 CASE_SECSAME(OverheadType::kU64, PrimaryType::kC64, uint64_t, complex64); in _mlir_ciface_newSparseTensor()
1673 CASE_SECSAME(OverheadType::kU64, PrimaryType::kC32, uint64_t, complex32); in _mlir_ciface_newSparseTensor()
1729 uint64_t isize = iref->sizes[0]; \ in FOREVERY_O()
1731 for (uint64_t r = 0; r < isize; r++) \ in FOREVERY_O()
1748 const uint64_t isize = iref->sizes[0]; \
1753 for (uint64_t r = 0; r < isize; r++) \
1838 void readSparseTensorShape(char *filename, std::vector<uint64_t> *out) { in readSparseTensorShape()
1844 const uint64_t rank = stfile.getRank(); in readSparseTensorShape()
1845 const uint64_t *dimSizes = stfile.getDimSizes(); in readSparseTensorShape()
1853 uint64_t rank, uint64_t nse, uint64_t *shape, V *values, \
1854 uint64_t *indices, uint64_t *perm, uint8_t *sparse) { \
1868 void convertFromMLIRSparseTensor##VNAME(void *tensor, uint64_t *pRank, \
1869 uint64_t *pNse, uint64_t **pShape, \
1870 V **pValues, uint64_t **pIndices) { \